예제 #1
0
def expandWildCards(filelist):
    """

    """
    l = GangaList()
    l.extend(iexpandWildCards(filelist))
    return l
예제 #2
0
def expandWildCards(filelist):
    """

    """
    l = GangaList()
    l.extend(iexpandWildCards(filelist))
    return l
예제 #3
0
    def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
        super(LHCbDataset, self).__init__()
        if files is None:
            files = []
        self.files = GangaList()
        process_files = True
        if fromRef:
            self.files._list.extend(files)
            process_files = False
        elif isinstance(files, GangaList):

            def isFileTest(_file):
                return isinstance(_file, IGangaFile)

            areFiles = all([isFileTest(f) for f in files._list])
            if areFiles:
                self.files._list.extend(files._list)
                process_files = False
        elif isinstance(files, LHCbDataset):
            self.files._list.extend(files.files._list)
            process_files = False

        if process_files:
            if isType(files, LHCbDataset):
                for this_file in files:
                    self.files.append(deepcopy(this_file))
            elif isType(files, IGangaFile):
                self.files.append(deepcopy(files))
            elif isType(files, (list, tuple, GangaList)):
                new_list = []
                for this_file in files:
                    if type(this_file) is str:
                        new_file = string_datafile_shortcut_lhcb(
                            this_file, None)
                    elif isType(this_file, IGangaFile):
                        new_file = stripProxy(this_file)
                    else:
                        new_file = strToDataFile(this_file)
                    new_list.append(new_file)
                self.files.extend(new_list)
            elif type(files) is str:
                self.files.append(string_datafile_shortcut_lhcb(files, None),
                                  False)
            else:
                raise GangaException(
                    "Unknown object passed to LHCbDataset constructor!")

        self.files._setParent(self)

        logger.debug("Processed inputs, assigning files")

        # Feel free to turn this on again for debugging but it's potentially quite expensive
        #logger.debug( "Creating dataset with:\n%s" % self.files )

        logger.debug("Assigned files")

        self.persistency = persistency
        self.depth = depth
        logger.debug("Dataset Created")
    def testPrintingGPIObjectList(self):

        g = GangaList()
        for _ in range(10):
            g.append(self._makeRandomTFile())

        g_string = str(g)
        assert eval(g_string) == g, 'String should correctly eval'
예제 #5
0
    def testPrintingGPIObjectList(self):

        g = GangaList()
        for _ in range(10):
            g.append(self._makeRandomTFile())

        g_string = str(g)
        assert eval(g_string) == g, 'String should correctly eval'
예제 #6
0
 def cloneVal(v):
     GangaList = _getGangaList()
     if isinstance(v, (list, tuple, GangaList)):
         new_v = GangaList()
         for elem in v:
             new_v.append(self.__cloneVal(elem, obj))
         return new_v
     else:
         return self.__cloneVal(v, obj)
    def testFullPrintingGPIObjectList(self):

        g = GangaList()
        for _ in range(10):
            g.append(self._makeRandomTFile())
        g_string = str(g)

        import StringIO
        sio = StringIO.StringIO()
        full_print(g, sio)
        assert g_string == str(sio.getvalue()).rstrip(), 'Orphaned lists should full_print'
예제 #8
0
파일: BOOT.py 프로젝트: MannyMoo/ganga
def getDiracFiles():
    import os
    from GangaDirac.Lib.Files.DiracFile import DiracFile
    from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
    filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns'
    logger.info('Creating list, this can take a while if you have a large number of SE files, please wait...')
    execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None)
    g = GangaList()
    with open(filename[1:], 'r') as lfnlist:
        lfnlist.seek(0)
        g.extend((DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines()))
    return addProxy(g)
예제 #9
0
    def testFullPrintingGPIObjectList(self):

        g = GangaList()
        for _ in range(10):
            g.append(self._makeRandomTFile())
        g_string = str(g)

        import StringIO
        sio = StringIO.StringIO()
        full_print(g, sio)
        assert g_string == str(
            sio.getvalue()).rstrip(), 'Orphaned lists should full_print'
예제 #10
0
    def setUp(self):
        self.ganga_list = GangaList()

        self.plain1 = [self._makeRandomTFile() for _ in range(15)]
        self.plain2 = [self._makeRandomTFile() for _ in range(10)]

        self.proxied1 = GangaList()
        self.proxied1.extend(self.plain1[:])
        self.proxied2 = GangaList()
        self.proxied2.extend(self.plain2[:])

        assert len(getProxyAttr(self.proxied1, '_list')) == len(self.plain1), 'Somthings wrong with construction 1'
        assert len(getProxyAttr(self.proxied2, '_list')) == len(self.plain2), 'Somthings wrong with construction 2'
예제 #11
0
    def testPrintingPlainList(self):

        g = GangaList()
        l = []
        print('"'+str(g)+'"')
        print('"'+str(l)+'"')
        print(l == g)
        assert str(l) == str(g), 'Empty lists should print the same'

        for i in xrange(100):
            g.append(i)
            l.append(i)
        assert str(l) == str(g), 'Normal Python objects should print the same'
예제 #12
0
    def testPrintingPlainList(self):

        g = GangaList()
        l = []
        print('"' + str(g) + '"')
        print('"' + str(l) + '"')
        print(l == g)
        assert str(l) == str(g), 'Empty lists should print the same'

        for i in xrange(100):
            g.append(i)
            l.append(i)
        assert str(l) == str(g), 'Normal Python objects should print the same'
예제 #13
0
파일: Proxy.py 프로젝트: alexpearce/ganga
 def _stripAttribute(obj, v, name):
     # just warn
     # print '**** checking',v,v.__class__,
     # isinstance(val,GPIProxyObject)
     if isinstance(v, list):
         from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
         v_new = GangaList()
         for elem in v:
             v_new.append(elem)
         v = v_new
     if isinstance(v, GPIProxyObject) or hasattr(v, implRef):
         v = stripProxy(v)
         logger.debug('%s property: assigned a component object (%s used)' % (name, implRef))
     return stripProxy(obj)._attribute_filter__set__(name, v)
예제 #14
0
파일: TestCopy.py 프로젝트: slangrock/ganga
    def testCopy(self):

        gl = GangaList()

        numberOfFiles = 100

        for _ in range(numberOfFiles):
            # add something which is generally not allowed by GangaList
            gl.append([self._makeRandomTFile()])

        assert len(gl) == numberOfFiles, 'Right number of files must be made'

        gl2 = copy.copy(gl)
        assert len(gl2) == len(gl), 'lists must be equal'
        assert gl2 is not gl, 'list must be copies'
예제 #15
0
파일: TestCopy.py 프로젝트: mjmottram/ganga
    def testCopy(self):

        gl = GangaList()

        numberOfFiles = 100

        for _ in range(numberOfFiles):
            # add something which is generally not allowed by GangaList
            gl.append([self._makeRandomTFile()])

        assert len(gl) == numberOfFiles, 'Right number of files must be made'

        gl2 = copy.copy(gl)
        assert gl2 == gl, 'lists must be equal'
        assert gl2 is not gl, 'list must be copies'
예제 #16
0
파일: BOOT.py 프로젝트: will-cern/ganga
def getDiracFiles():
    import os
    from GangaDirac.Lib.Files.DiracFile import DiracFile
    from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
    filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns'
    logger.info(
        'Creating list, this can take a while if you have a large number of SE files, please wait...'
    )
    execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None)
    g = GangaList()
    with open(filename[1:], 'r') as lfnlist:
        lfnlist.seek(0)
        g.extend(
            (DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines()))
    return addProxy(g)
예제 #17
0
    def setUp(self):
        self.ganga_list = GangaList()

        self.plain1 = [self._makeRandomTFile() for _ in range(15)]
        self.plain2 = [self._makeRandomTFile() for _ in range(10)]

        self.proxied1 = GangaList()
        self.proxied1.extend(self.plain1[:])
        self.proxied2 = GangaList()
        self.proxied2.extend(self.plain2[:])

        assert len(getProxyAttr(self.proxied1, '_list')) == len(
            self.plain1), 'Somthings wrong with construction 1'
        assert len(getProxyAttr(self.proxied2, '_list')) == len(
            self.plain2), 'Somthings wrong with construction 2'
예제 #18
0
파일: Proxy.py 프로젝트: slangrock/ganga
 def _stripAttribute(obj, v, name):
     # just warn
     # print '**** checking',v,v.__class__,
     # isinstance(val,GPIProxyObject)
     if isinstance(v, list):
         from Ganga.GPI import GangaList
         v_new = GangaList()
         for elem in v:
             v_new.append(elem)
         v = v_new
     if isinstance(v, GPIProxyObject) or hasattr(v, implRef):
         v = stripProxy(v)
         logger.debug('%s property: assigned a component object (%s used)' %
                      (name, implRef))
     return stripProxy(obj)._attribute_filter__set__(name, v)
예제 #19
0
파일: TestCopy.py 프로젝트: chrisburr/ganga
    def testDeepCopy(self):

        gl = GangaList()

        numberOfFiles = 100

        for _ in range(numberOfFiles):
            # add something which is generally not allowed by GangaList
            gl.append([self._makeRandomTFile()])

        assert len(gl) == numberOfFiles, "Right number of files must be made"

        gl2 = copy.deepcopy(gl)
        assert gl2 == gl, "lists must be equal"
        assert gl2 is not gl, "list must be copies"
        assert gl[0] is not gl2[0], "the references must be copied"
예제 #20
0
    def __cloneVal(self, v, obj):

        item = obj._schema[getName(self)]

        if v is None:
            if item.hasProperty('category'):
                assertion = item['optional'] and (item['category'] != 'internal')
            else:
                assertion = item['optional']
            #assert(assertion)
            if assertion is False:
                logger.warning("Item: '%s'. of class type: '%s'. Has a Default value of 'None' but is NOT optional!!!" % (getName(self), type(obj)))
                logger.warning("Please contact the developers and make sure this is updated!")
            return None
        elif isinstance(v, str):
            return str(v)
        elif isinstance(v, int):
            return int(v)
        elif isinstance(v, dict):
            new_dict = {}
            for key, item in new_dict.iteritems():
                new_dict[key] = self.__cloneVal(v, obj)
            return new_dict
        else:
            if not isinstance(v, Node) and isinstance(v, (list, tuple)):
                try:
                    GangaList = _getGangaList()
                    new_v = GangaList()
                except ImportError:
                    new_v = []
                for elem in v:
                    new_v.append(self.__cloneVal(elem, obj))
                #return new_v
            elif not isinstance(v, Node):
                if inspect.isclass(v):
                    new_v = v()
                else:
                    new_v = v
                if not isinstance(new_v, Node):
                    logger.error("v: %s" % str(v))
                    raise GangaException("Error: found Object: %s of type: %s expected an object inheriting from Node!" % (str(v), str(type(v))))
                else:
                    new_v = self.__copyNodeObject(new_v, obj)
            else:
                new_v = self.__copyNodeObject(v, obj)

            return new_v
예제 #21
0
    def testDeepCopy(self):

        from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
        gl = GangaList()

        numberOfFiles = 100

        for _ in range(numberOfFiles):
            # add something which is generally not allowed by GangaList
            gl.append([self._makeRandomTFile()])

        assert len(gl) == numberOfFiles, 'Right number of files must be made'

        gl2 = copy.deepcopy(gl)
        assert len(gl2) == len(gl), 'lists must be equal'
        assert gl2 is not gl, 'list must be copies'
        assert gl[0] is not gl2[0], 'the references must not be copied'
예제 #22
0
파일: TestCopy.py 프로젝트: Erni1619/ganga
    def testDeepCopy(self):

        from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
        gl = GangaList()

        numberOfFiles = 100

        for _ in range(numberOfFiles):
            # add something which is generally not allowed by GangaList
            gl.append([self._makeRandomTFile()])

        assert len(gl) == numberOfFiles, 'Right number of files must be made'

        gl2 = copy.deepcopy(gl)
        assert len(gl2) == len(gl), 'lists must be equal'
        assert gl2 is not gl, 'list must be copies'
        assert gl[0] is not gl2[0], 'the references must not be copied'
예제 #23
0
    def __init__(self, *args, **kwargs):
        super(TestGangaList, self).__init__(*args, **kwargs)

        self.plain1 = []
        self.plain2 = []

        self.proxied1 = []
        self.proxied2 = []

        self.ganga_list = GangaList()
예제 #24
0
    def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
        super(LHCbDataset, self).__init__()
        if files is None:
            files = []
        self.files = GangaList()
        process_files = True
        if fromRef:
            self.files._list.extend(files)
            process_files = False
        elif isinstance(files, GangaList):
            def isFileTest(_file):
                return isinstance(_file, IGangaFile)
            areFiles = all([isFileTest(f) for f in files._list])
            if areFiles:
                self.files._list.extend(files._list)
                process_files = False
        elif isinstance(files, LHCbDataset):
            self.files._list.extend(files.files._list)
            process_files = False

        if process_files:
            if isType(files, LHCbDataset):
                for this_file in files:
                    self.files.append(deepcopy(this_file))
            elif isType(files, IGangaFile):
                self.files.append(deepcopy(this_file))
            elif isType(files, (list, tuple, GangaList)):
                new_list = []
                for this_file in files:
                    if type(this_file) is str:
                        new_file = string_datafile_shortcut_lhcb(this_file, None)
                    elif isType(this_file, IGangaFile):
                        new_file = stripProxy(this_file)
                    else:
                        new_file = strToDataFile(this_file)
                    new_list.append(new_file)
                self.files.extend(new_list)
            elif type(files) is str:
                self.files.append(string_datafile_shortcut_lhcb(this_file, None), False)
            else:
                raise GangaException("Unknown object passed to LHCbDataset constructor!")

        self.files._setParent(self)

        logger.debug("Processed inputs, assigning files")

        # Feel free to turn this on again for debugging but it's potentially quite expensive
        #logger.debug( "Creating dataset with:\n%s" % self.files )
        
        logger.debug("Assigned files")

        self.persistency = persistency
        self.depth = depth
        logger.debug("Dataset Created")
예제 #25
0
    def setUp(self):
        super(TestGangaList, self).setUp()

        self.plain1 = [self._makeRandomTFile() for _ in range(15)]
        self.plain2 = [self._makeRandomTFile() for _ in range(10)]

        self.proxied1 = GangaList()
        self.proxied1.extend(self.plain1[:])
        self.proxied2 = GangaList()
        self.proxied2.extend(self.plain2[:])

        t = TFile()
        real_t = stripProxy(t)
        new_proxy_t = addProxy(real_t)
        #hopefully_t = stripProxy(new_proxy_t)
        #assert real_t is hopefully_t
        assert t is new_proxy_t

        self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction")
        self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction")
예제 #26
0
    def setUp(self):
        super(TestGangaList, self).setUp()

        self.plain1 = [self._makeRandomTFile() for _ in range(15)]
        self.plain2 = [self._makeRandomTFile() for _ in range(10)]

        self.proxied1 = GangaList()
        self.proxied1.extend(self.plain1[:])
        self.proxied2 = GangaList()
        self.proxied2.extend(self.plain2[:])

        t = TFile()
        real_t = stripProxy(t)
        new_proxy_t = addProxy(real_t)
        #hopefully_t = stripProxy(new_proxy_t)
        #assert real_t is hopefully_t
        assert t is new_proxy_t

        self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction")
        self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction")
예제 #27
0
    def __init__(self, files=None, persistency=None, depth=0):
        super(LHCbDataset, self).__init__()
        if files is None:
            files = []
        new_files = GangaList()
        if isType(files, LHCbDataset):
            for this_file in files:
                new_files.append(deepcopy(this_file))
        elif isType(files, IGangaFile):
            new_files.append(deepcopy(this_file))
        elif isType(files, (list, tuple, GangaList)):
            new_list = []
            for this_file in files:
                if type(this_file) is str:
                    new_file = string_datafile_shortcut_lhcb(this_file, None)
                elif isType(this_file, IGangaFile):
                    new_file = this_file
                else:
                    new_file = strToDataFile(this_file)
                new_list.append(stripProxy(new_file))
            stripProxy(new_files)._list = new_list
        elif type(files) is str:
            new_files.append(string_datafile_shortcut_lhcb(this_file, None), False)
        else:
            raise GangaException("Unknown object passed to LHCbDataset constructor!")
        new_files._setParent(self)

        logger.debug("Processed inputs, assigning files")

        # Feel free to turn this on again for debugging but it's potentially quite expensive
        #logger.debug( "Creating dataset with:\n%s" % files )
        self.files = new_files
        
        logger.debug("Assigned files")

        self.persistency = persistency
        self.depth = depth
        logger.debug("Dataset Created")
예제 #28
0
 def testNonZero(self):
     """@ExpectedFailure"""
     assert not GangaList(
     ), 'An empty GangaList should be false, just like a list'
예제 #29
0
    def put(self, lfn='', force=False, uploadSE="", replicate=False):
        """
        Try to upload file sequentially to storage elements defined in configDirac['allDiracSE'].
        File will be uploaded to the first SE that the upload command succeeds for.

        The file is uploaded to the SE described by the DiracFile.defaultSE attribute

        Alternatively, the user can specify an uploadSE which contains an SE
        which the file is to be uploaded to.

        If the user wants to replicate this file(s) across all SE then they should state replicate = True.

        Return value will be either the stdout from the dirac upload command if not
        using the wildcard characters '*?[]' in the namePattern.
        If the wildcard characters are used then the return value will be a list containing
        newly created DiracFile objects which were the result of glob-ing the wildcards.

        The objects in this list will have been uploaded or had their failureReason attribute populated if the
        upload failed.
        """

        if self.lfn != "" and force == False and lfn == '':
            logger.warning(
                "Warning you're about to 'put' this DiracFile: %s on the grid as it already has an lfn: %s"
                % (self.namePattern, self.lfn))
            decision = raw_input('y / [n]:')
            while not (decision in ['y', 'n'] or decision == ''):
                decision = raw_input('y / [n]:')

            if decision == 'y':
                pass
            else:
                return

        if (lfn != '' and self.lfn != '') and force == False:
            logger.warning(
                "Warning you're attempting to put this DiracFile: %s" %
                self.namePattern)
            logger.warning("It currently has an LFN associated with it: %s" %
                           self.lfn)
            logger.warning(
                "Do you want to continue and attempt to upload to: %s" % lfn)
            decision = raw_input('y / [n]:')
            while not (decision in ['y', 'n', '']):
                decision = raw_input('y / [n]:')

            if decision == 'y':
                pass
            else:
                return

        if lfn != '':
            self.lfn = lfn

        # looks like will only need this for the interactive uploading of jobs.
        # Also if any backend need dirac upload on client then when downloaded
        # this will upload then delete the file.

        if self.namePattern == "":
            if self.lfn != '':
                logger.warning(
                    "'Put'-ing a file with ONLY an existing LFN makes no sense!"
                )
            raise GangaException(
                'Can\'t upload a file without a local file name.')

        sourceDir = self.localDir
        if self.localDir is None:
            sourceDir = os.getcwd()
            # attached to a job, use the joboutputdir
            if self._parent != None and os.path.isdir(
                    self.getJobObject().outputdir):
                sourceDir = self.getJobObject().outputdir

        if not os.path.isdir(sourceDir):
            raise GangaException(
                'localDir attribute is not a valid dir, don\'t know from which dir to take the file'
            )

        if regex.search(self.namePattern) is not None:
            if self.lfn != "":
                logger.warning(
                    "Cannot specify a single lfn for a wildcard namePattern")
                logger.warning("LFN will be generated automatically")
                self.lfn = ""

        selfConstructedLFN = False

        import glob
        if self.remoteDir == '' and self.lfn == '':
            import datetime
            t = datetime.datetime.now()
            this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
            self.lfn = os.path.join(configDirac['DiracLFNBase'],
                                    'GangaFiles_%s' % this_date)
            selfConstructedLFN = True
        #if self.remoteDir == '' and self.lfn != '':
        #    self.remoteDir = configDirac['DiracLFNBase']

        if self.remoteDir[:4] == 'LFN:':
            lfn_base = self.remoteDir[4:]
        else:
            lfn_base = self.remoteDir

        if uploadSE != "":
            storage_elements = all_SE_list(uploadSE)
        else:
            storage_elements = all_SE_list(self.defaultSE)

        outputFiles = GangaList()
        for this_file in glob.glob(os.path.join(sourceDir, self.namePattern)):
            name = this_file

            if not os.path.exists(name):
                if not self.compressed:
                    raise GangaException(
                        'Cannot upload file. File "%s" must exist!' % name)
                name += '.gz'
                if not os.path.exists(name):
                    raise GangaException('File "%s" must exist!' % name)
            else:
                if self.compressed:
                    os.system('gzip -c %s > %s.gz' % (name, name))
                    name += '.gz'
                    if not os.path.exists(name):
                        raise GangaException('File "%s" must exist!' % name)

            if lfn == "":
                lfn = os.path.join(lfn_base, os.path.basename(name))

            if selfConstructedLFN is True:
                self.lfn = os.path.join(self.lfn, os.path.basename(name))

            lfn = self.lfn

            d = DiracFile()
            d.namePattern = os.path.basename(name)
            d.compressed = self.compressed
            d.localDir = sourceDir
            stderr = ''
            stdout = ''
            logger.info('Uploading file %s to %s as %s' %
                        (name, storage_elements[0], lfn))
            stdout = execute('uploadFile("%s", "%s", %s)' %
                             (lfn, name, str([storage_elements[0]])))
            if type(stdout) == str:
                logger.warning("Couldn't upload file '%s': %s" %
                               (os.path.basename(name), stdout))
                continue
            if stdout.get('OK', False) and lfn in stdout.get(
                    'Value', {'Successful': {}})['Successful']:
                # when doing the two step upload delete the temp file
                if self.compressed or self._parent != None:
                    os.remove(name)
                # need another eval as datetime needs to be included.
                guid = stdout['Value']['Successful'][lfn].get('GUID', '')
                if regex.search(self.namePattern) is not None:
                    d.lfn = lfn
                    d.remoteDir = os.path.dirname(lfn)
                    d.locations = stdout['Value']['Successful'][lfn].get(
                        'allDiracSE', '')
                    d.guid = guid
                    outputFiles.append(GPIProxyObjectFactory(d))
                    continue
                else:
                    self.lfn = lfn
                    self.remoteDir = os.path.dirname(lfn)
                    self.locations = stdout['Value']['Successful'][lfn].get(
                        'allDiracSE', '')
                    self.guid = guid
                # return ## WHY?
            else:
                failureReason = "Error in uploading file %s : %s" % (
                    os.path.basename(name), str(stdout))
                logger.error(failureReason)
                if regex.search(self.namePattern) is not None:
                    d.failureReason = failureReason
                    outputFiles.append(GPIProxyObjectFactory(d))
                    continue
                self.failureReason = failureReason
                return str(stdout)

        if replicate == True:

            if len(outputFiles) == 1 or len(outputFiles) == 0:
                storage_elements.pop(0)
                for se in storage_elements:
                    self.replicate(se)
            else:
                storage_elements.pop(0)
                for this_file in outputFiles:
                    for se in storage_elements:
                        this_file.replicate(se)

        if len(outputFiles) > 0:
            return GPIProxyObjectFactory(outputFiles)
        else:
            outputFiles.append(self)
            return GPIProxyObjectFactory(outputFiles)
예제 #30
0
class TestGangaList(GangaGPITestCase):
    def __init__(self):

        self.ganga_list = None

        self.plain1 = []
        self.plain2 = []

        self.proxied1 = []
        self.proxied2 = []

    def _makeRandomString(self):
        str_len = random.randint(3, 10)
        s = ''
        for _ in range(str_len):
            s += random.choice(string.ascii_letters)
        return s

    def _makeRandomTFile(self):
        name = self._makeRandomString()
        subdir = self._makeRandomString()
        return TFile(name=name, subdir=subdir)

    def setUp(self):
        self.ganga_list = GangaList()

        self.plain1 = [self._makeRandomTFile() for _ in range(15)]
        self.plain2 = [self._makeRandomTFile() for _ in range(10)]

        self.proxied1 = GangaList()
        self.proxied1.extend(self.plain1[:])
        self.proxied2 = GangaList()
        self.proxied2.extend(self.plain2[:])

        assert len(getProxyAttr(self.proxied1, '_list')) == len(
            self.plain1), 'Somthings wrong with construction 1'
        assert len(getProxyAttr(self.proxied2, '_list')) == len(
            self.plain2), 'Somthings wrong with construction 2'

    def testAllListMethodsExported(self):
        """Tests that all methods on list are exposed by GangaList"""
        def getmethods(_obj):
            # get all the method names for an object
            return dir(_obj)

        list_methods = getmethods([])
        gangalist_methods = getmethods(self.ganga_list)

        missing_methods = []
        for m in list_methods:
            if not m in gangalist_methods:
                missing_methods.append(m)

        if missing_methods:
            logger.info(missing_methods)

        assert not missing_methods, \
            'Not all of lists methods are implemented: %s' % str(
                missing_methods)

    def testEq(self):
        """Tests the equality op that the rest of the tests rely on."""
        assert self.proxied1 == self.plain1, 'Proxied and non-proxied versions should be the same'

    def testNq(self):
        assert self.proxied1 != None
        assert not self.proxied1 != self.proxied1, 'Lists are the same'
        assert self.proxied1 != self.proxied2, 'Lists are different'
        assert self.plain1 != self.proxied2, 'Lists are different'

    def testNonZero(self):
        """@ExpectedFailure"""
        assert not GangaList(
        ), 'An empty GangaList should be false, just like a list'

    def testAdd(self):
        """Test __add__"""
        assert len(self.plain1 + self.plain2) == len(self.proxied1 +
                                                     self.proxied2)
        assert isProxy(self.proxied1 + self.proxied2)

    def testAddMixed(self):
        """Test __add__ with mixed lists and GangaLists"""
        assert len(self.plain1 + self.plain2) == len(self.proxied1 +
                                                     self.plain2)
        assert len(self.plain2 + self.plain1) == len(self.plain2 +
                                                     self.proxied1)
        assert isProxy(self.proxied1 + self.plain2)

        assert len(self.plain2 + self.plain1) == len(self.plain2 +
                                                     self.proxied1)
        assert len(self.plain1 + self.plain2) == len(self.plain1 +
                                                     self.proxied2)
        assert isinstance(self.plain1 + self.proxied2, list)

    def testAddMixed2(self):

        self.plain1 = range(10)
        self.plain2 = range(10)

        assert isProxy(self.proxied2[-1]), 'Element access must get proxies'
        assert not isProxy(self.plain1[0]), 'Element access must not proxies'
        assert isProxy(
            (self.plain1 +
             self.proxied2)[-1]), 'File objects should remain proxies'
        assert not isProxy(
            (self.plain1 +
             self.proxied2)[0]), 'Objects in plain lists should be left alone'

        assert (self.plain1 + self.proxied2
                )[-1] == self.proxied2[-1], 'File objects should be equal'
        assert (self.plain1 + self.proxied2
                )[-1] is self.proxied2[-1], 'File objects should be identical'

    def testAddStr(self):
        """Makes sure that only lists can be added."""
        try:
            [] + ''
            assert False, 'Line above should throw a TypeError'
        except TypeError:
            pass

        try:
            self.proxied1 + ''
            assert False, 'Line above should throw a TypeError'
        except TypeError:
            pass

    def testContains(self):
        """Tests __contains__"""

        plist = [addProxy(x) for x in self.plain1]
        assert plist == self.proxied1

        for p in plist:
            assert isProxy(p)
            assert p in self.proxied1, 'Proxied list should contain each proxied object'

    def testDelItem(self):
        """Test __delitem__"""

        for p in [addProxy(x) for x in self.plain1[:]]:
            assert isProxy(p)
            del self.proxied1[self.proxied1.index(p)]

    def testGE(self):
        """Test __ge__"""

        assert (self.plain1 >= self.plain2) == (
            self.proxied1 >=
            self.proxied2), 'The lists should have the same ge'
        assert (self.plain2 >= self.plain1) == (
            self.proxied2 >=
            self.proxied1), 'The lists should have the same ge'

        assert (self.proxied1 >= self.proxied2) != (
            self.proxied2 >= self.proxied1), 'The gt should invert correctly'

    def testGetItem(self):
        """Test __getitem__"""

        for i in range(len(self.proxied1)):
            assert isProxy(self.proxied1[i])

    def testGetSlice(self):
        """Test __getslice__"""

        slices = [(0, 0), (0, len(self.plain1))]

        for s in slices:
            assert self.plain1[s[0]:s[1]] == self.proxied1[
                s[0]:s[1]], 'Slices %s should be the same' % str(s)

        t = self.plain1[:]
        assert t is not self.plain1, 'Slice should be a copy.'
        assert self.plain1[:] is not t

        t = self.proxied1[:]
        assert t is not self.proxied1, 'Slice should be a copy.'
        assert self.proxied1[:] is not t

    def testGT(self):
        """Test __gt__"""

        assert (self.plain1 > self.plain2) == (
            self.proxied1 > self.proxied2), 'The lists should have the same gt'
        assert (self.plain2 > self.plain1) == (
            self.proxied2 > self.proxied1), 'The lists should have the same gt'

        assert (self.proxied1 > self.proxied2) != (
            self.proxied2 > self.proxied1), 'The gt should invert correctly'

    def testIAdd(self):
        """Test __iadd__"""
        assert isProxy(self.proxied1), 'Must be Proxy'
        assert isProxy(self.proxied2), 'Must be Proxy'

        self.plain1 += self.plain2
        self.proxied1 += self.proxied2

        assert self.plain1 == self.proxied1, 'Addition should be the same'
        assert isProxy(self.proxied1), 'Proxy must be added'

    def testIAddMixed(self):
        """Test __iadd__ where we mix lists and GangaLists"""
        assert isProxy(self.proxied1), 'Must be Proxy'
        assert isProxy(self.proxied2), 'Must be Proxy'

        self.plain1 += self.plain2
        self.proxied1 += self.plain2

        assert self.plain1 == self.proxied1, 'Addition should be the same'
        assert isProxy(self.proxied1), 'Proxy must be added'
        assert isinstance(self.plain1, list), 'Must be list instance'

        self.plain2 += self.proxied1
        self.proxied2 += self.proxied1

        assert self.plain2 == self.proxied2, 'Addition should be the same'
        assert isProxy(self.proxied2), 'Proxy must be added'
        assert isinstance(self.plain2, list), 'Must be list instance'

    def testIAddSlice(self):
        """Test __iadd__ on slices"""

        s1 = self.plain1[3:7]
        s1 += self.plain2[2:5]

        s2 = self.proxied1[3:7]
        s2 += self.proxied2[2:5]

        assert s1 == s2, 'Addition should be the same'

        assert not isProxy(s1), 'Proxy Not Needed'
        assert isProxy(s2), 'Proxy Needed'

    def testIdentity(self):
        """Tests we obey list like identity relations."""

        t = self.plain1[4]
        assert self.plain1[4] is t

        t = self.proxied1[4]
        assert self.proxied1[4] is t

    def testIMul(self):
        """Test __imul__"""

        self.plain1 *= 5
        self.proxied1 *= 5

        assert self.plain1 == self.proxied1, 'Multiplication should be the same'
        assert isProxy(self.proxied1), 'Proxy must be added'

    def testIMulSlice(self):
        """Test __imul__ on slices"""

        s1 = self.plain1[3:7]
        s1 *= 6

        s2 = self.proxied1[3:7]
        s2 *= 6

        assert s1 == s2, 'Addition should be the same'

    def testLE(self):
        """Test __le__"""

        assert (self.plain1 <= self.plain2) == (
            self.proxied1 <=
            self.proxied2), 'The lists should have the same le'
        assert (self.plain2 <= self.plain1) == (
            self.proxied2 <=
            self.proxied1), 'The lists should have the same le'

        assert (self.proxied1 <= self.proxied2) != (
            self.proxied2 <= self.proxied1), 'The le should invert correctly'

    def testLen(self):
        """Tests __len__"""
        assert len(self.plain1) == len(
            self.proxied1), 'Lengths should be the same'

    def testLT(self):
        """Test __lt__"""

        assert (self.plain1 < self.plain2) == (
            self.proxied1 < self.proxied2), 'The lists should have the same lt'
        assert (self.plain2 < self.plain1) == (
            self.proxied2 < self.proxied1), 'The lists should have the same lt'

        assert (self.proxied1 < self.proxied2) != (
            self.proxied2 < self.proxied1), 'The lt should invert correctly'

    def testMul(self):
        """Test __mul__"""
        assert len(self.plain1 * 7) == len(self.proxied1 * 7)
        assert isProxy(self.proxied1 * 9)

        for p in self.proxied1:
            assert isProxy(p)

    def testNE(self):
        """Test __ne__"""

        assert self.plain1 != self.plain2
        assert self.proxied1 != self.proxied2, 'Lists should be different'

        assert self.plain1[0:5] != self.plain1[2:7]
        assert self.proxied1[0:5] != self.proxied1[
            2:7], 'Lists should be different'

    def testRMul(self):
        """Test __rmul__"""

        t1 = 5 * self.plain1
        t2 = 5 * self.proxied1

        assert len(t1) == len(t2), 'Multiplication should be the same'

    def testReversed(self):
        """Test the __reversed__ feature (new in python 2.4)."""

        from TFile import TFile as tF
        count = len(self.proxied1) - 1
        for i in self.proxied1.__reversed__():
            assert i is self.proxied1[count]
            assert isProxy(i)
            assert isType(i, tF)
            count -= 1

    def testSetItem(self):
        """Test __setitem__"""

        t = TFile(name='foo', subdir='cheese')
        test_index = 7
        self.plain1[test_index] = t
        assert self.plain1[test_index] is t

        self.proxied1[test_index] = t
        assert self.proxied1[test_index] is t
        assert isProxy(self.proxied1[test_index])

    def testSetSliceProxyList(self):

        self.plain1[3:7] = self.plain2[3:7]
        assert self.plain1[3:7] == self.plain2[
            3:7], 'The lists should be equal'
        assert self.plain1[3:7] is not self.plain2[
            3:7], 'The lists should be copies'
        self.plain1[4] = self.plain2[9]
        assert self.plain1[4] != self.plain2[4]

        tmp = self.plain1[2:9]
        assert self.plain1[2:9] is not tmp

        self.proxied1[3:7] = self.proxied2[3:7]
        assert self.proxied1[3:7] == self.proxied2[
            3:7], 'The lists should be equal'
        assert self.proxied1[3:7] is not self.proxied2[
            3:7], 'The lists should be copies'
        self.proxied1[4] = self.proxied2[9]
        assert self.proxied1[4] != self.proxied2[4]

        tmp = self.proxied1[2:9]
        assert self.proxied1[2:9] is not tmp

    def testAppend(self):

        t = addProxy(TFile(name='foo'))

        self.plain1.append(t)
        assert self.plain1[-1] == t
        assert self.plain1.pop() == t

        self.proxied1.append(t)
        assert self.proxied1[-1] == t
        assert self.proxied1[-1] is t, 'Identity Test'
        assert isProxy(self.proxied1[-1]), 'Make sure we get back a proxy'
        assert self.proxied1.pop() == t

    def testExtend(self):

        t1 = [self._makeRandomTFile() for _ in xrange(10)]

        self.plain1.extend(t1)
        self.proxied1.extend(t1)

        assert self.plain1 == self.proxied1, 'Lists should be the same'

        t2 = self.proxied1[4:7]
        assert isProxy(t2)
        from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
        assert isType(t2, glist)
        self.plain1.extend(t2)
        self.proxied1.extend(t2)
        assert self.plain1 == self.proxied1, 'Lists should be the same'

    def testIndex(self):

        t = addProxy(TFile(name='foo'))
        self.proxied1.insert(8, t)
        assert self.proxied1[8] == t
        assert self.proxied1.index(t) == 8

    def testInsert(self):

        t = addProxy(TFile(name='foo'))
        self.proxied1.insert(8, t)
        assert self.proxied1[8] == t

    def testPop(self):

        list_len = len(self.proxied1)

        t = self.proxied1[-1]
        r = self.proxied1.pop()
        assert t == r
        assert t is r
        assert len(self.proxied1) == list_len - 1
        assert t not in self.proxied1
        assert t._impl not in self.proxied1._impl

        t = self.proxied1[6]
        r = self.proxied1.pop(6)
        assert t == r
        assert t is r
        assert len(self.proxied1) == list_len - 2
        assert t not in self.proxied1
        assert t._impl not in self.proxied1._impl

    def testRemove(self):

        t = addProxy(TFile(name='bar'))
        self.proxied1.insert(7, t)
        list_len = len(self.proxied1)

        self.proxied1.remove(t)

        assert len(self.proxied1) == list_len - 1
        assert t not in self.proxied1
        assert t._impl not in self.proxied1._impl

    def testIter(self):

        from TFile import TFile as tF

        count = 0
        for f in self.proxied1:
            count += 1
            assert isProxy(f)
            assert isType(f, tF)

        assert count == len(self.proxied1), 'Must visit every member'

    def testCmp(self):

        assert cmp(self.proxied1,
                   self.proxied2) == cmp(self.plain1, self.plain2)

    def testHash(self):

        try:
            hash(int(self.proxied1))
            assert False, 'Lists are not hashable'
        except TypeError:
            pass
예제 #31
0
파일: Objects.py 프로젝트: kreczko/ganga
    def __set__(self, _obj, _val):

        obj = stripProxy(_obj)
        val = stripProxy(_val)

        from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList

        cs = self._bind_method(obj, self._checkset_name)
        if cs:
            cs(val)
        this_filter = self._bind_method(obj, self._filter_name)
        if this_filter:
            val = this_filter(val)

        # LOCKING
        obj._getWriteAccess()

        # self._check_getter()

        item = stripProxy(obj._schema[self._name])

        def cloneVal(v):
            return self.__cloneVal(v, obj)

        obj_reg = obj._getRegistry()
        full_reg = obj_reg is not None and hasattr(obj_reg, 'dirty_flush_counter')

        ### THIS CODE HERE CHANGES THE DIRTY FLUSH COUNTER SUCH THAT GANGALIST OBJECTS ARE WRITTEN TO DISK ATOMICALLY
        ### THIS COMES WITH A MAJOR PERFORMANCE IMPROVEMENT IN SOME CODE THAT REALLY SHOULDN'T BE SLOW
        ### Maybe delete this if/when we aren't using XML repo in future as allowing a really dirty repo is probably bad m'kay

        ## NB Should really tie in the default here to defaults from Core
        old_count = 10
        if hasattr(val, '__len__') and full_reg:
            old_count = obj_reg.dirty_flush_counter
            val_len = 2*len(val) + 10
            obj_reg.dirty_flush_counter = val_len
            obj_len = 1
            if len(val) > 0:
                bare_val = stripProxy(val)
                if hasattr(bare_val, '_schema'):
                    obj_len = len(stripProxy(bare_val._schema).datadict.keys())
                    obj_len = obj_len*2
            val_len = val_len * obj_len

        if item['sequence']:
            _preparable = True if item['preparable'] else False
            if len(val) == 0:
                val = GangaList()
            else:
                if isType(item, Schema.ComponentItem):
                        val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
                else:
                        val = makeGangaList(val, parent=obj, preparable=_preparable)
        else:
            if isType(item, Schema.ComponentItem):
                val = cloneVal(val)

        if hasattr(val, '_setParent'):
            val._setParent(obj)
        
        ### RESET DIRTY COUNT
        if full_reg:
            obj_reg.dirty_flush_counter = old_count

        obj._data[self._name] = val

        obj._setDirty()
예제 #32
0
class LHCbDataset(GangaDataset):

    '''Class for handling LHCb data sets (i.e. inputdata for LHCb jobs).

    Example Usage:
    ds = LHCbDataset(["lfn:/some/lfn.file","pfn:/some/pfn.file"])
    ds[0] # DiracFile("/some/lfn.file") - see DiracFile docs for usage
    ds[1] # PhysicalFile("/some/pfn.file")- see PhysicalFile docs for usage
    len(ds) # 2 (number of files)
    ds.getReplicas() # returns replicas for *all* files in the data set
    ds.replicate("CERN-USER") # replicate *all* LFNs to "CERN-USER" SE
    ds.getCatalog() # returns XML catalog slice
    ds.optionsString() # returns Gaudi-sytle options 
    [...etc...]
    '''
    schema = {}
    docstr = 'List of PhysicalFile and DiracFile objects'
    schema['files'] = GangaFileItem(defvalue=[], typelist=['str', 'Ganga.GPIDev.Adapters.IGangaFile.IGangaFile'], sequence=1, doc=docstr)
    docstr = 'Ancestor depth to be queried from the Bookkeeping'
    schema['depth'] = SimpleItem(defvalue=0, doc=docstr)
    docstr = 'Use contents of file rather than generating catalog.'
    schema['XMLCatalogueSlice'] = GangaFileItem(defvalue=None, doc=docstr)
    docstr = 'Specify the dataset persistency technology'
    schema['persistency'] = SimpleItem(
        defvalue=None, typelist=['str', 'type(None)'], doc=docstr)
    schema['treat_as_inputfiles'] = SimpleItem(defvalue=False, doc="Treat the inputdata as inputfiles, i.e. copy the inputdata to the WN")

    _schema = Schema(Version(3, 0), schema)
    _category = 'datasets'
    _name = "LHCbDataset"
    _exportmethods = ['getReplicas', '__len__', '__getitem__', 'replicate',
                      'hasLFNs', 'append', 'extend', 'getCatalog', 'optionsString',
                      'getLFNs', 'getFileNames', 'getFullFileNames',
                      'difference', 'isSubset', 'isSuperset', 'intersection',
                      'symmetricDifference', 'union', 'bkMetadata',
                      'isEmpty', 'hasPFNs', 'getPFNs']  # ,'pop']

    def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
        super(LHCbDataset, self).__init__()
        if files is None:
            files = []
        self.files = GangaList()
        process_files = True
        if fromRef:
            self.files._list.extend(files)
            process_files = False
        elif isinstance(files, GangaList):
            def isFileTest(_file):
                return isinstance(_file, IGangaFile)
            areFiles = all([isFileTest(f) for f in files._list])
            if areFiles:
                self.files._list.extend(files._list)
                process_files = False
        elif isinstance(files, LHCbDataset):
            self.files._list.extend(files.files._list)
            process_files = False

        if process_files:
            if isType(files, LHCbDataset):
                for this_file in files:
                    self.files.append(deepcopy(this_file))
            elif isType(files, IGangaFile):
                self.files.append(deepcopy(this_file))
            elif isType(files, (list, tuple, GangaList)):
                new_list = []
                for this_file in files:
                    if type(this_file) is str:
                        new_file = string_datafile_shortcut_lhcb(this_file, None)
                    elif isType(this_file, IGangaFile):
                        new_file = stripProxy(this_file)
                    else:
                        new_file = strToDataFile(this_file)
                    new_list.append(new_file)
                self.files.extend(new_list)
            elif type(files) is str:
                self.files.append(string_datafile_shortcut_lhcb(this_file, None), False)
            else:
                raise GangaException("Unknown object passed to LHCbDataset constructor!")

        self.files._setParent(self)

        logger.debug("Processed inputs, assigning files")

        # Feel free to turn this on again for debugging but it's potentially quite expensive
        #logger.debug( "Creating dataset with:\n%s" % self.files )
        
        logger.debug("Assigned files")

        self.persistency = persistency
        self.depth = depth
        logger.debug("Dataset Created")

    def __getitem__(self, i):
        '''Proivdes scripting (e.g. ds[2] returns the 3rd file) '''
        #this_file = self.files[i]
        # print type(this_file)
        # return this_file
        # return this_file
        # return this_file
        if type(i) == type(slice(0)):
            ds = LHCbDataset(files=self.files[i])
            ds.depth = self.depth
            #ds.XMLCatalogueSlice = self.XMLCatalogueSlice
            return ds
        else:
            return self.files[i]

    def getReplicas(self):
        'Returns the replicas for all files in the dataset.'
        lfns = self.getLFNs()
        cmd = 'getReplicas(%s)' % str(lfns)
        result = get_result(cmd, 'LFC query error. Could not get replicas.')
        return result['Successful']

    def hasLFNs(self):
        'Returns True is the dataset has LFNs and False otherwise.'
        for f in self.files:
            if isDiracFile(f):
                return True
        return False

    def hasPFNs(self):
        'Returns True is the dataset has PFNs and False otherwise.'
        for f in self.files:
            if not isDiracFile(f):
                return True
        return False

    def replicate(self, destSE=''):
        '''Replicate all LFNs to destSE.  For a list of valid SE\'s, type
        ds.replicate().'''

        if not destSE:
            from GangaDirac.Lib.Files.DiracFile import DiracFile
            DiracFile().replicate('')
            return
        if not self.hasLFNs():
            raise GangaException('Cannot replicate dataset w/ no LFNs.')

        retry_files = []

        for f in self.files:
            if not isDiracFile(f):
                continue
            try:
                result = f.replicate( destSE=destSE )
            except Exception as err:
                msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
                logger.warning(msg)
                logger.warning("Error: %s" % str(err))
                retry_files.append(f)

        for f in retry_files:
            try:
                result = f.replicate( destSE=destSE )
            except Exception as err:
                msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
                logger.warning(msg)
                logger.warning(str(err))

    def extend(self, files, unique=False):
        '''Extend the dataset. If unique, then only add files which are not
        already in the dataset.'''
        from Ganga.GPIDev.Base import ReadOnlyObjectError

        if self._parent is not None and self._parent._readonly():
            raise ReadOnlyObjectError('object Job#%s  is read-only and attribute "%s/inputdata" cannot be modified now' % (self._parent.id, getName(self)))

        _external_files = []

        if type(files) is str or isType(files, IGangaFile):
            _external_files = [files]
        elif type(files) in [list, tuple]:
            _external_files = files
        elif isType(files, LHCbDataset):
            _external_files = files.files
        else:
            if not hasattr(files, "__getitem__") or not hasattr(files, '__iter__'):
                _external_files = [files]

        # just in case they extend w/ self
        _to_remove = []
        for this_file in _external_files:
            if hasattr(this_file, 'subfiles'):
                if len(this_file.subfiles) > 0:
                    _external_files = makeGangaListByRef(this_file.subfiles)
                    _to_remove.append(this_file)
            if type(this_file) is str:
                _external_files.append(string_datafile_shortcut_lhcb(this_file, None))
                _to_remove.append(this_file)

        for _this_file in _to_remove:
            _external_files.pop(_external_files.index(_this_file))

        for this_f in _external_files:
            _file = getDataFile(this_f)
            if _file is None:
                _file = this_f
            myName = _file.namePattern
            from GangaDirac.Lib.Files.DiracFile import DiracFile
            if isType(_file, DiracFile):
                myName = _file.lfn
            if unique and myName in self.getFileNames():
                continue
            self.files.append(stripProxy(_file))

    def removeFile(self, input_file):
        try:
            self.files.remove(input_file)
        except:
            raise GangaException('Dataset has no file named %s' % input_file.namePattern)

    def getLFNs(self):
        'Returns a list of all LFNs (by name) stored in the dataset.'
        lfns = []
        if not self:
            return lfns
        for f in self.files:
            if isDiracFile(f):
                subfiles = f.getSubFiles()
                if len(subfiles) == 0:
                    lfns.append(f.lfn)
                else:
                    for file in subfiles:
                        lfns.append(file.lfn)

        #logger.debug( "Returning LFNS:\n%s" % str(lfns) )
        logger.debug("Returning #%s LFNS" % str(len(lfns)))
        return lfns

    def getPFNs(self):
        'Returns a list of all PFNs (by name) stored in the dataset.'
        pfns = []
        if not self:
            return pfns
        for f in self.files:
            if isPFN(f):
                pfns.append(f.namePattern)
        return pfns

    def getFullFileNames(self):
        'Returns all file names w/ PFN or LFN prepended.'
        names = []
        from GangaDirac.Lib.Files.DiracFile import DiracFile
        for f in self.files:
            if isType(f, DiracFile):
                names.append('LFN:%s' % f.lfn)
            else:
                try:
                    names.append('PFN:%s' % f.namePattern)
                except:
                    logger.warning("Cannot determine filename for: %s " % f)
                    raise GangaException("Cannot Get File Name")
        return names

    def getCatalog(self, site=''):
        '''Generates an XML catalog from the dataset (returns the XML string).
        Note: site defaults to config.LHCb.LocalSite
        Note: If the XMLCatalogueSlice attribute is set, then it returns
              what is written there.'''
        if hasattr(self.XMLCatalogueSlice, 'name'):
            if self.XMLCatalogueSlice.name:
                f = open(self.XMLCatalogueSlice.name)
                xml_catalog = f.read()
                f.close()
                return xml_catalog
        if not site:
            site = getConfig('LHCb')['LocalSite']
        lfns = self.getLFNs()
        depth = self.depth
        tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml')
        cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \
              % (str(lfns), depth, site, tmp_xml.name)
        result = get_result(cmd, 'LFN->PFN error. XML catalog error.')
        xml_catalog = tmp_xml.read()
        tmp_xml.close()
        return xml_catalog

    def optionsString(self, file=None):
        'Returns the Gaudi-style options string for the dataset (if a filename' \
            ' is given, the file is created and output is written there).'
        if not self or len(self) == 0:
            return ''
        snew = ''
        if self.persistency == 'ROOT':
            snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency
        elif self.persistency == 'POOL':
            snew = '\ntry:\n    #new method\n    from GaudiConf import IOExtension\n    IOExtension(\"%s\").inputFiles([' % self.persistency
        elif self.persistency == None:
            snew = '\ntry:\n    #new method\n    from GaudiConf import IOExtension\n    IOExtension().inputFiles(['
        else:
            logger.warning(
                "Unknown LHCbDataset persistency technology... reverting to None")
            snew = '\ntry:\n    #new method\n    from GaudiConf import IOExtension\n    IOExtension().inputFiles(['

        sold = '\nexcept ImportError:\n    #Use previous method\n    from Gaudi.Configuration import EventSelector\n    EventSelector().Input=['
        sdatasetsnew = ''
        sdatasetsold = ''

        dtype_str_default = getConfig('LHCb')['datatype_string_default']
        dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns']
        for f in self.files:
            dtype_str = dtype_str_default
            for this_str in dtype_str_patterns:
                matched = False
                for pat in dtype_str_patterns[this_str]:
                    if fnmatch.fnmatch(f.namePattern, pat):
                        dtype_str = this_str
                        matched = True
                        break
                if matched:
                    break
            sdatasetsnew += '\n        '
            sdatasetsold += '\n        '
            if isDiracFile(f):
                sdatasetsnew += """ \"LFN:%s\",""" % f.lfn
                sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn, dtype_str)
            else:
                sdatasetsnew += """ \"PFN:%s\",""" % f.namePattern
                sdatasetsold += """ \"DATAFILE='PFN:%s' %s\",""" % (f.namePattern, dtype_str)
        if sdatasetsold.endswith(","):
            if self.persistency == 'ROOT':
                sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)"""
            else:
                sdatasetsnew = sdatasetsnew[:-1] + """\n    ], clear=True)"""
            sdatasetsold = sdatasetsold[:-1]
            sdatasetsold += """\n    ]"""
        if(file):
            f = open(file, 'w')
            if self.persistency == 'ROOT':
                f.write(snew)
                f.write(sdatasetsnew)
            else:
                f.write(snew)
                f.write(sdatasetsnew)
                f.write(sold)
                f.write(sdatasetsold)
            f.close()
        else:
            if self.persistency == 'ROOT':
                return snew + sdatasetsnew
            else:
                return snew + sdatasetsnew + sold + sdatasetsold

    def _checkOtherFiles(self, other ):
        if isType(other, GangaList) or isType(other, []):
            other_files = LHCbDataset(other).getFullFileNames()
        elif isType(other, LHCbDataset):
            other_files = other.getFullFileNames()
        else:
            raise GangaException("Unknown type for difference")
        return other_files

    def difference(self, other):
        '''Returns a new data set w/ files in this that are not in other.'''
        other_files = self._checkOtherFiles(other)
        files = set(self.getFullFileNames()).difference(other_files)
        data = LHCbDataset()
        data.extend(list(files))
        data.depth = self.depth
        return data

    def isSubset(self, other):
        '''Is every file in this data set in other?'''
        other_files = self._checkOtherFiles(other)
        return set(self.getFileNames()).issubset(other_files)

    def isSuperset(self, other):
        '''Is every file in other in this data set?'''
        other_files = self._checkOtherFiles(other)
        return set(self.getFileNames()).issuperset(other_files)

    def symmetricDifference(self, other):
        '''Returns a new data set w/ files in either this or other but not
        both.'''
        other_files = other._checkOtherFiles(other)
        files = set(self.getFullFileNames()).symmetric_difference(other_files)
        data = LHCbDataset()
        data.extend(list(files))
        data.depth = self.depth
        return data

    def intersection(self, other):
        '''Returns a new data set w/ files common to this and other.'''
        other_files = other._checkOtherFiles(other)
        files = set(self.getFullFileNames()).intersection(other_files)
        data = LHCbDataset()
        data.extend(list(files))
        data.depth = self.depth
        return data

    def union(self, other):
        '''Returns a new data set w/ files from this and other.'''
        other_files = self._checkOtherFiles(other)
        files = set(self.getFullFileNames()).union(other_files)
        data = LHCbDataset()
        data.extend(list(files))
        data.depth = self.depth
        return data

    def bkMetadata(self):
        'Returns the bookkeeping metadata for all LFNs. '
        logger.info("Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info...")
        cmd = 'bkMetaData(%s)' % self.getLFNs()
        b = get_result(cmd, 'Error removing replica. Replica rm error.')
        return b
예제 #33
0
class TestGangaList(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super(TestGangaList, self).__init__(*args, **kwargs)

        self.plain1 = []
        self.plain2 = []

        self.proxied1 = []
        self.proxied2 = []

        self.ganga_list = GangaList()

    @staticmethod
    def _makeRandomString():
        str_len = random.randint(3, 10)
        s = ''
        for _ in range(str_len):
            s += random.choice(string.ascii_letters)
        return s

    @staticmethod
    def _makeRandomTFile():
        name = TestGangaList._makeRandomString()
        subdir = TestGangaList._makeRandomString()
        return TFile(name=name, subdir=subdir)

    def setUp(self):
        super(TestGangaList, self).setUp()

        self.plain1 = [self._makeRandomTFile() for _ in range(15)]
        self.plain2 = [self._makeRandomTFile() for _ in range(10)]

        self.proxied1 = GangaList()
        self.proxied1.extend(self.plain1[:])
        self.proxied2 = GangaList()
        self.proxied2.extend(self.plain2[:])

        t = TFile()
        real_t = stripProxy(t)
        new_proxy_t = addProxy(real_t)
        #hopefully_t = stripProxy(new_proxy_t)
        #assert real_t is hopefully_t
        assert t is new_proxy_t

        self.assertEqual(len(getProxyAttr(self.proxied1, '_list')),
                         len(self.plain1),
                         "Something's wrong with construction")
        self.assertEqual(len(getProxyAttr(self.proxied2, '_list')),
                         len(self.plain2),
                         "Something's wrong with construction")

    def testAllListMethodsExported(self):
        """Tests that all methods on list are exposed by GangaList"""
        def getmethods(_obj):
            # get all the method names for an object
            return dir(_obj)

        list_methods = getmethods([])
        gangalist_methods = getmethods(self.ganga_list)

        missing_methods = []
        for m in list_methods:
            if not m in gangalist_methods:
                missing_methods.append(m)

        if missing_methods:
            logger.info(missing_methods)

        self.assertFalse(
            missing_methods,
            'Not all of lists methods are implemented: %s' % missing_methods)

    def testEq(self):
        """Tests the equality op that the rest of the tests rely on."""
        self.assertEqual(
            self.proxied1, self.plain1,
            'Proxied and non-proxied versions should be the same')

    def testNq(self):
        self.assertIsNotNone(self.proxied1, None)
        self.assertFalse(self.proxied1 != self.proxied1, 'Lists are the same')
        self.assertNotEqual(self.proxied1, self.proxied2,
                            'Lists are different')
        self.assertNotEqual(self.plain1, self.proxied2, 'Lists are different')

    def testNonZero(self):
        """@ExpectedFailure"""
        self.assertFalse(
            GangaList(),
            'An empty GangaList should be false, just like a list')

    def testAdd(self):
        """Test __add__"""
        self.assertEqual(self.plain1 + self.plain2,
                         self.proxied1 + self.proxied2)
        self.assertTrue(isProxy(self.proxied1 + self.proxied2))

    def testAddMixed(self):
        """Test __add__ with mixed lists and GangaLists"""
        self.assertEqual((self.plain1 + self.plain2),
                         (self.proxied1 + self.plain2))
        self.assertEqual((self.plain2 + self.plain1),
                         (self.plain2 + self.proxied1))
        self.assertTrue(isProxy(self.proxied1 + self.plain2))

        self.assertEqual((self.plain2 + self.plain1),
                         (self.plain2 + self.proxied1))
        self.assertEqual((self.plain1 + self.plain2),
                         (self.plain1 + self.proxied2))
        self.assertTrue(isinstance(self.plain1 + self.proxied2, list))

    def testAddMixed2(self):

        self.plain1 = range(10)
        self.plain2 = range(10)

        self.assertTrue(isProxy(self.proxied2[-1]),
                        'Element access must get proxies')
        self.assertFalse(isProxy(self.plain1[0]),
                         'Element access must not proxies')
        self.assertTrue(isProxy((self.plain1 + self.proxied2)[-1]),
                        'File objects should remain proxies')
        self.assertFalse(isProxy((self.plain1 + self.proxied2)[0]),
                         'Objects in plain lists should be left alone')

        self.assertEqual((self.plain1 + self.proxied2)[-1], self.proxied2[-1],
                         'File objects should be equal')
        self.assertIs((self.plain1 + self.proxied2)[-1], self.proxied2[-1],
                      'File objects should be identical')

    def testAddStr(self):
        """Makes sure that only lists can be added."""
        try:
            [] + ''
            assert False, 'Line above should throw a TypeError'
        except TypeError:
            pass

        try:
            self.proxied1 + ''
            assert False, 'Line above should throw a TypeError'
        except TypeError:
            pass

    def testContains(self):
        """Tests __contains__"""

        plist = [addProxy(x) for x in self.plain1]
        self.assertEqual(plist, self.proxied1)

        for p in plist:
            self.assertTrue(isProxy(p))
            self.assertIn(p, self.proxied1,
                          'Proxied list should contain each proxied object')

    def testDelItem(self):
        """Test __delitem__"""

        for p in [addProxy(x) for x in self.plain1[:]]:
            self.assertTrue(isProxy(p))
            del self.proxied1[self.proxied1.index(p)]

    def testGE(self):
        """Test __ge__"""

        self.assertEqual((self.plain1 >= self.plain2),
                         (self.proxied1 >= self.proxied2),
                         'The lists should have the same ge')
        self.assertEqual((self.plain2 >= self.plain1),
                         (self.proxied2 >= self.proxied1),
                         'The lists should have the same ge')

        self.assertNotEqual((self.proxied1 >= self.proxied2),
                            (self.proxied2 >= self.proxied1),
                            'The gt should invert correctly')

    def testGetItem(self):
        """Test __getitem__"""

        for i in range(len(self.proxied1)):
            self.assertTrue(isProxy(self.proxied1[i]))

    def testGetSlice(self):
        """Test __getslice__"""

        slices = [(0, 0), (0, len(self.plain1))]

        for s in slices:
            self.assertEqual(self.plain1[s[0]:s[1]], self.proxied1[s[0]:s[1]],
                             'Slices {0} should be the same'.format(s))

        t = self.plain1[:]
        self.assertIsNot(t, self.plain1, 'Slice should be a copy.')
        self.assertIsNot(self.plain1[:], t)

        t = self.proxied1[:]
        self.assertIsNot(t, self.proxied1, 'Slice should be a copy.')
        self.assertIsNot(self.proxied1[:], t)

    def testGT(self):
        """Test __gt__"""

        self.assertEqual((self.plain1 > self.plain2),
                         (self.proxied1 > self.proxied2),
                         'The lists should have the same gt')
        self.assertEqual((self.plain2 > self.plain1),
                         (self.proxied2 > self.proxied1),
                         'The lists should have the same gt')

        self.assertNotEqual((self.proxied1 > self.proxied2),
                            (self.proxied2 > self.proxied1),
                            'The gt should invert correctly')

    def testIAdd(self):
        """Test __iadd__"""
        self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
        self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')

        self.plain1 += self.plain2
        self.proxied1 += self.proxied2

        self.assertEqual(self.plain1, self.proxied1,
                         'Addition should be the same')
        self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')

    def testIAddMixed(self):
        """Test __iadd__ where we mix lists and GangaLists"""
        self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
        self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')

        self.plain1 += self.plain2
        self.proxied1 += self.plain2

        self.assertEqual(self.plain1, self.proxied1,
                         'Addition should be the same')
        self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
        self.assertTrue(isinstance(self.plain1, list), 'Must be list instance')

        self.plain2 += self.proxied1
        self.proxied2 += self.proxied1

        self.assertEqual(self.plain2, self.proxied2,
                         'Addition should be the same')
        self.assertTrue(isProxy(self.proxied2), 'Proxy must be added')
        self.assertTrue(isinstance(self.plain2, list), 'Must be list instance')

    def testIAddSlice(self):
        """Test __iadd__ on slices"""

        s1 = self.plain1[3:7]
        s1 += self.plain2[2:5]

        s2 = self.proxied1[3:7]
        s2 += self.proxied2[2:5]

        self.assertEqual(s1, s2, 'Addition should be the same')

        self.assertFalse(isProxy(s1), 'Proxy Not Needed')
        self.assertTrue(isProxy(s2), 'Proxy Needed')

    def testIdentity(self):
        """Tests we obey list like identity relations."""

        t = self.plain1[4]
        self.assertIs(self.plain1[4], t)

        t = self.proxied1[4]
        self.assertIs(self.proxied1[4], t)

    def testIMul(self):
        """Test __imul__"""

        self.plain1 *= 5
        self.proxied1 *= 5

        self.assertEqual(self.plain1, self.proxied1,
                         'Multiplication should be the same')
        self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')

    def testIMulSlice(self):
        """Test __imul__ on slices"""

        s1 = self.plain1[3:7]
        s1 *= 6

        s2 = self.proxied1[3:7]
        s2 *= 6

        self.assertEqual(s1, s2, 'Addition should be the same')

    def testLE(self):
        """Test __le__"""

        self.assertEqual(self.plain1 <= self.plain2,
                         self.proxied1 <= self.proxied2,
                         'The lists should have the same le')
        self.assertEqual(self.plain2 <= self.plain1,
                         self.proxied2 <= self.proxied1,
                         'The lists should have the same le')

        self.assertNotEqual(self.proxied1 <= self.proxied2,
                            self.proxied2 <= self.proxied1,
                            'The le should invert correctly')

    def testLen(self):
        """Tests __len__"""
        self.assertEqual(len(self.plain1), len(self.proxied1),
                         'Lengths should be the same')

    def testLT(self):
        """Test __lt__"""

        self.assertEqual(self.plain1 < self.plain2,
                         self.proxied1 < self.proxied2,
                         'The lists should have the same lt')
        self.assertEqual(self.plain2 < self.plain1,
                         self.proxied2 < self.proxied1,
                         'The lists should have the same lt')

        self.assertNotEqual(self.proxied1 < self.proxied2,
                            self.proxied2 < self.proxied1,
                            'The lt should invert correctly')

    def testMul(self):
        """Test __mul__"""
        self.assertEqual((self.plain1 * 7), (self.proxied1 * 7))
        self.assertTrue(isProxy(self.proxied1 * 9))

        for p in self.proxied1:
            self.assertTrue(isProxy(p))

    def testNE(self):
        """Test __ne__"""

        self.assertNotEqual(self.plain1, self.plain2)
        self.assertNotEqual(self.proxied1, self.proxied2,
                            'Lists should be different')

        self.assertNotEqual(self.plain1[0:5], self.plain1[2:7])
        self.assertNotEqual(self.proxied1[0:5], self.proxied1[2:7],
                            'Lists should be different')

    def testRMul(self):
        """Test __rmul__"""

        t1 = 5 * self.plain1
        t2 = 5 * self.proxied1

        self.assertEqual(t1, t2, 'Multiplication should be the same')

    def testReversed(self):
        """Test the __reversed__ feature (new in python 2.4)."""

        count = len(self.proxied1) - 1
        for i in self.proxied1.__reversed__():
            self.assertIs(i, self.proxied1[count])
            self.assertTrue(isProxy(i))
            self.assertTrue(isType(i, TFile))
            count -= 1

    def testSetItem(self):
        """Test __setitem__"""

        t = TFile(name='foo', subdir='cheese')
        test_index = 7
        self.plain1[test_index] = t
        self.assertIs(self.plain1[test_index], t)

        self.proxied1[test_index] = t
        self.assertIs(self.proxied1[test_index], t)
        self.assertTrue(isProxy(self.proxied1[test_index]))

    def testSetSliceProxyList(self):

        self.plain1[3:7] = self.plain2[3:7]
        self.assertEqual(self.plain1[3:7], self.plain2[3:7],
                         'The lists should be equal')
        self.assertIsNot(self.plain1[3:7], self.plain2[3:7],
                         'The lists should be copies')
        self.plain1[4] = self.plain2[9]
        self.assertNotEqual(self.plain1[4], self.plain2[4])

        tmp = self.plain1[2:9]
        self.assertIsNot(self.plain1[2:9], tmp)

        self.proxied1[3:7] = self.proxied2[3:7]
        self.assertEqual(self.proxied1[3:7], self.proxied2[3:7],
                         'The lists should be equal')
        self.assertIsNot(self.proxied1[3:7], self.proxied2[3:7],
                         'The lists should be copies')
        self.proxied1[4] = self.proxied2[9]
        self.assertNotEqual(self.proxied1[4], self.proxied2[4])

        tmp = self.proxied1[2:9]
        self.assertIsNot(self.proxied1[2:9], tmp)

    def testAppend(self):

        t = TFile(name='foo')

        self.plain1.append(t)
        self.assertEqual(self.plain1[-1], t)
        self.assertEqual(self.plain1.pop(), t)

        self.proxied1.append(t)
        self.assertEqual(self.proxied1[-1], t)
        self.assertIs(self.proxied1[-1], t, 'Identity Test')
        self.assertTrue(isProxy(self.proxied1[-1]),
                        'Make sure we get back a proxy')
        self.assertEqual(self.proxied1.pop(), t)

    def testExtend(self):

        t1 = [self._makeRandomTFile() for _ in xrange(10)]

        self.plain1.extend(t1)
        self.proxied1.extend(t1)

        self.assertEqual(self.plain1, self.proxied1,
                         'Lists should be the same')

        t2 = self.proxied1[4:7]
        self.assertTrue(isProxy(t2))
        from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
        self.assertTrue(isType(t2, glist))
        self.plain1.extend(t2)
        self.proxied1.extend(t2)
        self.assertEqual(self.plain1, self.proxied1,
                         'Lists should be the same')

    def testIndex(self):

        t = TFile(name='foo')
        self.proxied1.insert(8, t)
        self.assertEqual(self.proxied1[8], t)
        self.assertEqual(self.proxied1.index(t), 8)

    def testInsert(self):

        t = TFile(name='foo')
        self.proxied1.insert(8, t)
        self.assertEqual(self.proxied1[8], t)

    def testPop(self):

        list_len = len(self.proxied1)

        t = self.proxied1[-1]
        r = self.proxied1.pop()
        self.assertEqual(t, r)
        self.assertIs(t, r)
        self.assertEqual(len(self.proxied1), list_len - 1)
        self.assertNotIn(t, self.proxied1)
        self.assertNotIn(t._impl, self.proxied1._impl)

        t = self.proxied1[6]
        r = self.proxied1.pop(6)
        self.assertEqual(t, r)
        self.assertIs(t, r)
        self.assertEqual(len(self.proxied1), list_len - 2)
        self.assertNotIn(t, self.proxied1)
        self.assertNotIn(t._impl, self.proxied1._impl)

    def testRemove(self):

        t = TFile(name='bar')
        self.proxied1.insert(7, t)
        list_len = len(self.proxied1)

        self.proxied1.remove(t)

        self.assertEqual(len(self.proxied1), list_len - 1)
        self.assertNotIn(t, self.proxied1)
        self.assertNotIn(t._impl, self.proxied1._impl)

    def testIter(self):

        count = 0
        for f in self.proxied1:
            count += 1
            self.assertTrue(isProxy(f))
            self.assertTrue(isType(f, TFile))

        self.assertEqual(count, len(self.proxied1), 'Must visit every member')

    def testCmp(self):

        self.assertEqual(cmp(self.proxied1, self.proxied2),
                         cmp(self.plain1, self.plain2))

    def testHash(self):

        try:
            hash(int(self.proxied1))
            assert False, 'Lists are not hashable'
        except TypeError:
            pass
예제 #34
0
    def __atomic_set__(self, _obj, _val):
        ## self: attribute being changed or Ganga.GPIDev.Base.Objects.Descriptor in which case getName(self) gives the name of the attribute being changed
        ## _obj: parent class which 'owns' the attribute
        ## _val: value of the attribute which we're about to set

        #if hasattr(_obj, getName(self)):
        #    if not isinstance(getattr(_obj, getName(self)), GangaObject):
        #        if type( getattr(_obj, getName(self)) ) == type(_val):
        #            object.__setattr__(_obj, getName(self), deepcopy(_val))
        #            return
#
#        if not isinstance(_obj, GangaObject) and type(_obj) == type(_val):
#            _obj = deepcopy(_val)
#            return

        obj = _obj
        temp_val = _val

        from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList

        if hasattr(obj, '_checkset_name'):
            checkSet = self._bind_method(obj, self._checkset_name)
            if checkSet is not None:
                checkSet(temp_val)
        if hasattr(obj, '_filter_name'):
            this_filter = self._bind_method(obj, self._filter_name)
            if this_filter:
                val = this_filter(temp_val)
            else:
                val = temp_val
        else:
            val = temp_val

        # LOCKING
        obj._getWriteAccess()

        #self._check_getter()

        item = obj._schema[getName(self)]

        def cloneVal(v):
            GangaList = _getGangaList()
            if isinstance(v, (list, tuple, GangaList)):
                new_v = GangaList()
                for elem in v:
                    new_v.append(self.__cloneVal(elem, obj))
                return new_v
            else:
                return self.__cloneVal(v, obj)

        ## If the item has been defined as a sequence great, let's continue!
        if item['sequence']:
            _preparable = True if item['preparable'] else False
            if len(val) == 0:
                GangaList = _getGangaList()
                new_val = GangaList()
            else:
                if isinstance(item, Schema.ComponentItem):
                    new_val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
                else:
                    new_val = makeGangaList(val, parent=obj, preparable=_preparable)
        else:
            ## Else we need to work out what we've got.
            if isinstance(item, Schema.ComponentItem):
                GangaList = _getGangaList()
                if isinstance(val, (list, tuple, GangaList)):
                    ## Can't have a GangaList inside a GangaList easily so lets not
                    if isinstance(_obj, GangaList):
                        newListObj = []
                    else:
                        newListObj = GangaList()

                    self.__createNewList(newListObj, val, cloneVal)
                    #for elem in val:
                    #    newListObj.append(cloneVal(elem))
                    new_val = newListObj
                else:
                    new_val = cloneVal(val)
            else:
                new_val = val
                pass
            #val = deepcopy(val)

        if isinstance(new_val, Node):
            new_val._setParent(obj)

        obj.setNodeAttribute(getName(self), new_val)

        obj._setDirty()
예제 #35
0
 def __init__(self, files=[], persistency=None, depth=0):
     new_files = GangaList()
     if isType(files, LHCbDataset):
         for this_file in files:
             new_files.append(copy.deepcopy(this_file))
     elif isType(files, IGangaFile):
         new_files.append(copy.deepcopy(this_file))
     elif type(files) == type([]):
         for this_file in files:
             if type(this_file) == type(''):
                 new_files.append(string_datafile_shortcut_lhcb(this_file, None), False)
             elif isType(this_file, IGangaFile):
                 new_files.append(this_file, False)
             else:
                 new_files.append(strToDataFile(this_file))
     elif type(files) == type(''):
         new_files.append(string_datafile_shortcut_lhcb(this_file, None), False)
     else:
         from Ganga.Core.exceptions import GangaException
         raise GangaException("Unknown object passed to LHCbDataset constructor!")
     new_files._setParent(self)
     super(LHCbDataset, self).__init__()
     # Feel free to turn this on again for debugging but it's potentially quite expensive
     #logger.debug( "Creating dataset with:\n%s" % files )
     self.files = new_files
     self.persistency = persistency
     self.depth = depth
     logger.debug("Dataset Created")
예제 #36
0
    def put(self, lfn='', force=False, uploadSE="", replicate=False):
        """
        Try to upload file sequentially to storage elements defined in configDirac['allDiracSE'].
        File will be uploaded to the first SE that the upload command succeeds for.

        The file is uploaded to the SE described by the DiracFile.defaultSE attribute

        Alternatively, the user can specify an uploadSE which contains an SE
        which the file is to be uploaded to.

        If the user wants to replicate this file(s) across all SE then they should state replicate = True.

        Return value will be either the stdout from the dirac upload command if not
        using the wildcard characters '*?[]' in the namePattern.
        If the wildcard characters are used then the return value will be a list containing
        newly created DiracFile objects which were the result of glob-ing the wildcards.

        The objects in this list will have been uploaded or had their failureReason attribute populated if the
        upload failed.
        """

        if self.lfn != "" and force == False and lfn == '':
            logger.warning(
                "Warning you're about to 'put' this DiracFile: %s on the grid as it already has an lfn: %s"
                % (self.namePattern, self.lfn))
            decision = raw_input('y / [n]:')
            while not (decision.lower() in ['y', 'n']
                       or decision.lower() == ''):
                decision = raw_input('y / [n]:')

            if decision.lower() == 'y':
                pass
            else:
                return

        if (lfn != '' and self.lfn != '') and force == False:
            logger.warning(
                "Warning you're attempting to put this DiracFile: %s" %
                self.namePattern)
            logger.warning("It currently has an LFN associated with it: %s" %
                           self.lfn)
            logger.warning(
                "Do you want to continue and attempt to upload to: %s" % lfn)
            decision = raw_input('y / [n]:')
            while not (decision.lower() in ['y', 'n', '']):
                decision = raw_input('y / [n]:')

            if decision.lower() == 'y':
                pass
            else:
                return

        if lfn and os.path.basename(lfn) != self.namePattern:
            logger.warning(
                "Changing namePattern from: '%s' to '%s' during put operation"
                % (self.namePattern, os.path.basename(lfn)))

        if lfn:
            self.lfn = lfn

        # looks like will only need this for the interactive uploading of jobs.
        # Also if any backend need dirac upload on client then when downloaded
        # this will upload then delete the file.

        if self.namePattern == "":
            if self.lfn != '':
                logger.warning(
                    "'Put'-ing a file with ONLY an existing LFN makes no sense!"
                )
            raise GangaFileError(
                'Can\'t upload a file without a local file name.')

        sourceDir = self.localDir
        if self.localDir is None:
            sourceDir = os.getcwd()
            # attached to a job, use the joboutputdir
            if self._parent != None and os.path.isdir(
                    self.getJobObject().outputdir):
                sourceDir = self.getJobObject().outputdir

        if not os.path.isdir(sourceDir):
            raise GangaFileError(
                'localDir attribute is not a valid dir, don\'t know from which dir to take the file'
            )

        if regex.search(self.namePattern) is not None:
            if self.lfn != "":
                logger.warning(
                    "Cannot specify a single lfn for a wildcard namePattern")
                logger.warning("LFN will be generated automatically")
                self.lfn = ""

        if not self.remoteDir:
            try:
                job = self.getJobObject()
                lfn_folder = os.path.join("GangaJob_%s" % job.getFQID('/'),
                                          "OutputFiles")
            except AssertionError:
                t = datetime.datetime.now()
                this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
                lfn_folder = os.path.join('GangaFiles_%s' % this_date)
            lfn_base = os.path.join(
                DiracFile.diracLFNBase(self.credential_requirements),
                lfn_folder)

        else:
            lfn_base = os.path.join(
                DiracFile.diracLFNBase(self.credential_requirements),
                self.remoteDir)

        if uploadSE == "":
            if self.defaultSE != "":
                storage_elements = [self.defaultSE]
            else:
                if configDirac['allDiracSE']:
                    storage_elements = [
                        random.choice(configDirac['allDiracSE'])
                    ]
                else:
                    raise GangaFileError(
                        "Can't upload a file without a valid defaultSE or storageSE, please provide one"
                    )
        elif isinstance(uploadSE, list):
            storage_elements = uploadSE
        else:
            storage_elements = [uploadSE]

        outputFiles = GangaList()
        for this_file in glob.glob(os.path.join(sourceDir, self.namePattern)):
            name = this_file

            if not os.path.exists(name):
                if not self.compressed:
                    raise GangaFileError(
                        'Cannot upload file. File "%s" must exist!' % name)
                name += '.gz'
                if not os.path.exists(name):
                    raise GangaFileError('File "%s" must exist!' % name)
            else:
                if self.compressed:
                    os.system('gzip -c %s > %s.gz' % (name, name))
                    name += '.gz'
                    if not os.path.exists(name):
                        raise GangaFileError('File "%s" must exist!' % name)

            lfn = os.path.join(lfn_base, os.path.basename(this_file))

            d = DiracFile()
            d.namePattern = os.path.basename(name)
            d.compressed = self.compressed
            d.localDir = sourceDir
            stderr = ''
            stdout = ''
            logger.info('Uploading file \'%s\' to \'%s\' as \'%s\'' %
                        (name, storage_elements[0], lfn))
            logger.debug('execute: uploadFile("%s", "%s", %s)' %
                         (lfn, os.path.join(sourceDir,
                                            name), str([storage_elements[0]])))
            try:
                stdout = execute('uploadFile("%s", "%s", %s)' %
                                 (lfn, os.path.join(sourceDir, name),
                                  str([storage_elements[0]])),
                                 cred_req=self.credential_requirements)
            except GangaDiracError as err:
                logger.warning("Couldn't upload file '%s': \'%s\'" %
                               (os.path.basename(name), err))
                failureReason = "Error in uploading file '%s' : '%s'" % (
                    os.path.basename(name), err)
                if regex.search(self.namePattern) is not None:
                    d.failureReason = failureReason
                    outputFiles.append(d)
                    continue
                self.failureReason += '\n' + failureReason
                continue

            stdout_temp = stdout.get('Successful')

            if not stdout_temp:
                msg = "Couldn't upload file '%s': \'%s\'" % (
                    os.path.basename(name), stdout)
                logger.warning(msg)
                if regex.search(self.namePattern) is not None:
                    d.failureReason = msg
                    outputFiles.append(d)
                    continue
                self.failureReason = msg
                continue
            else:
                lfn_out = stdout_temp[lfn]

            # when doing the two step upload delete the temp file
            if self.compressed or self._parent != None:
                os.remove(name)
            # need another eval as datetime needs to be included.
            guid = lfn_out.get('GUID', '')
            if regex.search(self.namePattern) is not None:
                d.lfn = lfn
                d.remoteDir = os.path.dirname(lfn)
                d.locations = lfn_out.get('allDiracSE', '')
                d.guid = guid
                outputFiles.append(d)
                continue
            else:
                self.lfn = lfn
                self.remoteDir = os.path.dirname(lfn)
                self.locations = lfn_out.get('allDiracSE', '')
                self.guid = guid

        if replicate == True:

            if len(outputFiles) == 1 or len(outputFiles) == 0:
                storage_elements.pop(0)
                for se in storage_elements:
                    self.replicate(se)
            else:
                storage_elements.pop(0)
                for this_file in outputFiles:
                    for se in storage_elements:
                        this_file.replicate(se)

        if len(outputFiles) > 0:
            return outputFiles
        else:
            outputFiles.append(self)
            return outputFiles
예제 #37
0
    def _create_subjob(self, job, dataset):
        logger.debug("_create_subjob")
        datatmp = []

        logger.debug("dataset size: %s" % str(len(dataset)))
        #logger.debug( "dataset: %s" % str(dataset) )

        from GangaLHCb.Lib.LHCbDataset.LHCbDataset import LHCbDataset

        if isinstance(dataset, LHCbDataset):
            for i in dataset:
                if isType(i, DiracFile):
                    datatmp.append(i)
                else:
                    logger.error(
                        "Unkown file-type %s, cannot perform split with file %s"
                        % (type(i), str(i)))
                    from Ganga.Core.exceptions import GangaException
                    raise GangaException(
                        "Unkown file-type %s, cannot perform split with file %s"
                        % (type(i), str(i)))
        elif type(dataset) == type([]) or isType(dataset, GangaList()):
            for this_file in dataset:
                if type(this_file) is str:
                    datatmp.append(allComponentFilters['gangafiles'](this_file,
                                                                     None))
                elif isType(this_file, IGangaFile):
                    datatmp.append(this_file)
                else:
                    logger.error("Unexpected type: %s" % str(type(this_file)))
                    logger.error("Wanted object to inherit from type: %s: %s" %
                                 (str(type(IGangaFile()))))
                    from Ganga.Core.exceptions import GangaException
                    x = GangaException("Unknown(unexpected) file object: %s" %
                                       this_file)
                    raise x
        elif type(dataset) is str:
            datatmp.append(DiracFile(lfn=dataset))
        else:
            logger.error("Unkown dataset type, cannot perform split here")
            from Ganga.Core.exceptions import GangaException
            logger.error("Dataset found: " + str(dataset))
            raise GangaException(
                "Unkown dataset type, cannot perform split here")

        logger.debug("Creating new Job in Splitter")
        j = Job()
        logger.debug("Copying From Job")
        j.copyFrom(
            stripProxy(job),
            ['splitter', 'subjobs', 'inputdata', 'inputsandbox', 'inputfiles'])
        logger.debug("Unsetting Splitter")
        j.splitter = None
        #logger.debug("Unsetting Merger")
        #j.merger = None
        #j.inputsandbox = [] ## master added automatically
        #j.inputfiles = []
        logger.debug("Setting InputData")
        j.inputdata = LHCbDataset(files=datatmp[:],
                                  persistency=self.persistency,
                                  depth=self.depth)
        #j.inputdata.XMLCatalogueSlice = self.XMLCatalogueSlice
        logger.debug("Returning new subjob")
        return j
예제 #38
0
class TestGangaList(GangaGPITestCase):

    def __init__(self):

        self.ganga_list = None

        self.plain1 = []
        self.plain2 = []

        self.proxied1 = []
        self.proxied2 = []

    def _makeRandomString(self):
        str_len = random.randint(3, 10)
        s = ''
        for _ in range(str_len):
            s += random.choice(string.ascii_letters)
        return s

    def _makeRandomTFile(self):
        name = self._makeRandomString()
        subdir = self._makeRandomString()
        return TFile(name=name, subdir=subdir)

    def setUp(self):
        self.ganga_list = GangaList()

        self.plain1 = [self._makeRandomTFile() for _ in range(15)]
        self.plain2 = [self._makeRandomTFile() for _ in range(10)]

        self.proxied1 = GangaList()
        self.proxied1.extend(self.plain1[:])
        self.proxied2 = GangaList()
        self.proxied2.extend(self.plain2[:])

        assert len(getProxyAttr(self.proxied1, '_list')) == len(
            self.plain1), 'Somthings wrong with construction'
        assert len(getProxyAttr(self.proxied2, '_list')) == len(
            self.plain2), 'Somthings wrong with construction'

    def testAllListMethodsExported(self):
        """Tests that all methods on list are exposed by GangaList"""

        def getmethods(_obj):
            # get all the method names for an object
            return dir(_obj)

        list_methods = getmethods([])
        gangalist_methods = getmethods(self.ganga_list)

        missing_methods = []
        for m in list_methods:
            if not m in gangalist_methods:
                missing_methods.append(m)

        if missing_methods:
            logger.info(missing_methods)

        assert not missing_methods, \
            'Not all of lists methods are implemented: %s' % str(
                missing_methods)

    def testEq(self):
        """Tests the equality op that the rest of the tests rely on."""
        assert self.proxied1 == self.plain1, 'Proxied and non-proxied versions should be the same'

    def testNq(self):
        assert self.proxied1 != None
        assert not self.proxied1 != self.proxied1, 'Lists are the same'
        assert self.proxied1 != self.proxied2, 'Lists are different'
        assert self.plain1 != self.proxied2, 'Lists are different'

    def testNonZero(self):
        """@ExpectedFailure"""
        assert not GangaList(
        ), 'An empty GangaList should be false, just like a list'

    def testAdd(self):
        """Test __add__"""
        assert (self.plain1 + self.plain2) == (self.proxied1 + self.proxied2)
        assert isProxy(self.proxied1 + self.proxied2)

    def testAddMixed(self):
        """Test __add__ with mixed lists and GangaLists"""
        assert (self.plain1 + self.plain2) == (self.proxied1 + self.plain2)
        assert (self.plain2 + self.plain1) == (self.plain2 + self.proxied1)
        assert isProxy(self.proxied1 + self.plain2)

        assert (self.plain2 + self.plain1) == (self.plain2 + self.proxied1)
        assert (self.plain1 + self.plain2) == (self.plain1 + self.proxied2)
        assert isinstance(self.plain1 + self.proxied2, list)

    def testAddMixed2(self):

        self.plain1 = range(10)
        self.plain2 = range(10)

        assert isProxy(self.proxied2[-1]), 'Element access must get proxies'
        assert not isProxy(self.plain1[0]), 'Element access must not proxies'
        assert isProxy(
            (self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies'
        assert not isProxy(
            (self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone'

        assert (
            self.plain1 + self.proxied2)[-1] == self.proxied2[-1], 'File objects should be equal'
        assert (
            self.plain1 + self.proxied2)[-1] is self.proxied2[-1], 'File objects should be identical'

    def testAddStr(self):
        """Makes sure that only lists can be added."""
        try:
            [] + ''
            assert False, 'Line above should throw a TypeError'
        except TypeError:
            pass

        try:
            self.proxied1 + ''
            assert False, 'Line above should throw a TypeError'
        except TypeError:
            pass

    def testContains(self):
        """Tests __contains__"""

        plist = [addProxy(x) for x in self.plain1]
        assert plist == self.proxied1

        for p in plist:
            assert isProxy(p)
            assert p in self.proxied1, 'Proxied list should contain each proxied object'

    def testDelItem(self):
        """Test __delitem__"""

        for p in [addProxy(x) for x in self.plain1[:]]:
            assert isProxy(p)
            del self.proxied1[self.proxied1.index(p)]

    def testGE(self):
        """Test __ge__"""

        assert (self.plain1 >= self.plain2) == (
            self.proxied1 >= self.proxied2), 'The lists should have the same ge'
        assert (self.plain2 >= self.plain1) == (
            self.proxied2 >= self.proxied1), 'The lists should have the same ge'

        assert (self.proxied1 >= self.proxied2) != (
            self.proxied2 >= self.proxied1), 'The gt should invert correctly'

    def testGetItem(self):
        """Test __getitem__"""

        for i in range(len(self.proxied1)):
            assert isProxy(self.proxied1[i])

    def testGetSlice(self):
        """Test __getslice__"""

        slices = [(0, 0), (0, len(self.plain1))]

        for s in slices:
            assert self.plain1[s[0]:s[1]] == self.proxied1[
                s[0]:s[1]], 'Slices %s should be the same' % str(s)

        t = self.plain1[:]
        assert t is not self.plain1, 'Slice should be a copy.'
        assert self.plain1[:] is not t

        t = self.proxied1[:]
        assert t is not self.proxied1, 'Slice should be a copy.'
        assert self.proxied1[:] is not t

    def testGT(self):
        """Test __gt__"""

        assert (self.plain1 > self.plain2) == (
            self.proxied1 > self.proxied2), 'The lists should have the same gt'
        assert (self.plain2 > self.plain1) == (
            self.proxied2 > self.proxied1), 'The lists should have the same gt'

        assert (self.proxied1 > self.proxied2) != (
            self.proxied2 > self.proxied1), 'The gt should invert correctly'

    def testIAdd(self):
        """Test __iadd__"""
        assert isProxy(self.proxied1), 'Must be Proxy'
        assert isProxy(self.proxied2), 'Must be Proxy'

        self.plain1 += self.plain2
        self.proxied1 += self.proxied2

        assert self.plain1 == self.proxied1, 'Addition should be the same'
        assert isProxy(self.proxied1), 'Proxy must be added'

    def testIAddMixed(self):
        """Test __iadd__ where we mix lists and GangaLists"""
        assert isProxy(self.proxied1), 'Must be Proxy'
        assert isProxy(self.proxied2), 'Must be Proxy'

        self.plain1 += self.plain2
        self.proxied1 += self.plain2

        assert self.plain1 == self.proxied1, 'Addition should be the same'
        assert isProxy(self.proxied1), 'Proxy must be added'
        assert isinstance(self.plain1, list), 'Must be list instance'

        self.plain2 += self.proxied1
        self.proxied2 += self.proxied1

        assert self.plain2 == self.proxied2, 'Addition should be the same'
        assert isProxy(self.proxied2), 'Proxy must be added'
        assert isinstance(self.plain2, list), 'Must be list instance'

    def testIAddSlice(self):
        """Test __iadd__ on slices"""

        s1 = self.plain1[3:7]
        s1 += self.plain2[2:5]

        s2 = self.proxied1[3:7]
        s2 += self.proxied2[2:5]

        assert s1 == s2, 'Addition should be the same'

        assert not isProxy(s1), 'Proxy Not Needed'
        assert isProxy(s2), 'Proxy Needed'

    def testIdentity(self):
        """Tests we obey list like identity relations."""

        t = self.plain1[4]
        assert self.plain1[4] is t

        t = self.proxied1[4]
        assert self.proxied1[4] is t

    def testIMul(self):
        """Test __imul__"""

        self.plain1 *= 5
        self.proxied1 *= 5

        assert self.plain1 == self.proxied1, 'Multiplication should be the same'
        assert isProxy(self.proxied1), 'Proxy must be added'

    def testIMulSlice(self):
        """Test __imul__ on slices"""

        s1 = self.plain1[3:7]
        s1 *= 6

        s2 = self.proxied1[3:7]
        s2 *= 6

        assert s1 == s2, 'Addition should be the same'

    def testLE(self):
        """Test __le__"""

        assert (self.plain1 <= self.plain2) == (
            self.proxied1 <= self.proxied2), 'The lists should have the same le'
        assert (self.plain2 <= self.plain1) == (
            self.proxied2 <= self.proxied1), 'The lists should have the same le'

        assert (self.proxied1 <= self.proxied2) != (
            self.proxied2 <= self.proxied1), 'The le should invert correctly'

    def testLen(self):
        """Tests __len__"""
        assert len(self.plain1) == len(
            self.proxied1), 'Lengths should be the same'

    def testLT(self):
        """Test __lt__"""

        assert (self.plain1 < self.plain2) == (
            self.proxied1 < self.proxied2), 'The lists should have the same lt'
        assert (self.plain2 < self.plain1) == (
            self.proxied2 < self.proxied1), 'The lists should have the same lt'

        assert (self.proxied1 < self.proxied2) != (
            self.proxied2 < self.proxied1), 'The lt should invert correctly'

    def testMul(self):
        """Test __mul__"""
        assert (self.plain1 * 7) == (self.proxied1 * 7)
        assert isProxy(self.proxied1 * 9)

        for p in self.proxied1:
            assert isProxy(p)

    def testNE(self):
        """Test __ne__"""

        assert self.plain1 != self.plain2
        assert self.proxied1 != self.proxied2, 'Lists should be different'

        assert self.plain1[0:5] != self.plain1[2:7]
        assert self.proxied1[0:5] != self.proxied1[
            2:7], 'Lists should be different'

    def testRMul(self):
        """Test __rmul__"""

        t1 = 5 * self.plain1
        t2 = 5 * self.proxied1

        assert t1 == t2, 'Multiplication should be the same'

    def testReversed(self):
        """Test the __reversed__ feature (new in python 2.4)."""

        from TFile import TFile as tF
        count = len(self.proxied1) - 1
        for i in self.proxied1.__reversed__():
            assert i is self.proxied1[count]
            assert isProxy(i)
            assert isType(i, tF)
            count -= 1

    def testSetItem(self):
        """Test __setitem__"""

        t = TFile(name='foo', subdir='cheese')
        test_index = 7
        self.plain1[test_index] = t
        assert self.plain1[test_index] is t

        self.proxied1[test_index] = t
        assert self.proxied1[test_index] is t
        assert isProxy(self.proxied1[test_index])

    def testSetSliceProxyList(self):

        self.plain1[3:7] = self.plain2[3:7]
        assert self.plain1[3:7] == self.plain2[
            3:7], 'The lists should be equal'
        assert self.plain1[3:7] is not self.plain2[
            3:7], 'The lists should be copies'
        self.plain1[4] = self.plain2[9]
        assert self.plain1[4] != self.plain2[4]

        tmp = self.plain1[2:9]
        assert self.plain1[2:9] is not tmp

        self.proxied1[3:7] = self.proxied2[3:7]
        assert self.proxied1[3:7] == self.proxied2[
            3:7], 'The lists should be equal'
        assert self.proxied1[3:7] is not self.proxied2[
            3:7], 'The lists should be copies'
        self.proxied1[4] = self.proxied2[9]
        assert self.proxied1[4] != self.proxied2[4]

        tmp = self.proxied1[2:9]
        assert self.proxied1[2:9] is not tmp

    def testAppend(self):

        t = addProxy(TFile(name='foo'))

        self.plain1.append(t)
        assert self.plain1[-1] == t
        assert self.plain1.pop() == t

        self.proxied1.append(t)
        assert self.proxied1[-1] == t
        assert self.proxied1[-1] is t, 'Identity Test'
        assert isProxy(self.proxied1[-1]), 'Make sure we get back a proxy'
        assert self.proxied1.pop() == t

    def testExtend(self):

        t1 = [self._makeRandomTFile() for _ in xrange(10)]

        self.plain1.extend(t1)
        self.proxied1.extend(t1)

        assert self.plain1 == self.proxied1, 'Lists should be the same'

        t2 = self.proxied1[4:7]
        assert isProxy(t2)
        from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
        assert isType(t2, glist)
        self.plain1.extend(t2)
        self.proxied1.extend(t2)
        assert self.plain1 == self.proxied1, 'Lists should be the same'

    def testIndex(self):

        t = addProxy(TFile(name='foo'))
        self.proxied1.insert(8, t)
        assert self.proxied1[8] == t
        assert self.proxied1.index(t) == 8

    def testInsert(self):

        t = addProxy(TFile(name='foo'))
        self.proxied1.insert(8, t)
        assert self.proxied1[8] == t

    def testPop(self):

        list_len = len(self.proxied1)

        t = self.proxied1[-1]
        r = self.proxied1.pop()
        assert t == r
        assert t is r
        assert len(self.proxied1) == list_len - 1
        assert t not in self.proxied1
        assert t._impl not in self.proxied1._impl

        t = self.proxied1[6]
        r = self.proxied1.pop(6)
        assert t == r
        assert t is r
        assert len(self.proxied1) == list_len - 2
        assert t not in self.proxied1
        assert t._impl not in self.proxied1._impl

    def testRemove(self):

        t = addProxy(TFile(name='bar'))
        self.proxied1.insert(7, t)
        list_len = len(self.proxied1)

        self.proxied1.remove(t)

        assert len(self.proxied1) == list_len - 1
        assert t not in self.proxied1
        assert t._impl not in self.proxied1._impl

    def testIter(self):

        from TFile import TFile as tF

        count = 0
        for f in self.proxied1:
            count += 1
            assert isProxy(f)
            assert isType(f, tF)

        assert count == len(self.proxied1), 'Must visit every member'

    def testCmp(self):

        assert cmp(self.proxied1, self.proxied2) == cmp(
            self.plain1, self.plain2)

    def testHash(self):

        try:
            hash(int(self.proxied1))
            assert False, 'Lists are not hashable'
        except TypeError:
            pass
예제 #39
0
파일: DiracFile.py 프로젝트: Erni1619/ganga
    def put(self, lfn='', force=False, uploadSE="", replicate=False):
        """
        Try to upload file sequentially to storage elements defined in configDirac['allDiracSE'].
        File will be uploaded to the first SE that the upload command succeeds for.

        The file is uploaded to the SE described by the DiracFile.defaultSE attribute

        Alternatively, the user can specify an uploadSE which contains an SE
        which the file is to be uploaded to.

        If the user wants to replicate this file(s) across all SE then they should state replicate = True.

        Return value will be either the stdout from the dirac upload command if not
        using the wildcard characters '*?[]' in the namePattern.
        If the wildcard characters are used then the return value will be a list containing
        newly created DiracFile objects which were the result of glob-ing the wildcards.

        The objects in this list will have been uploaded or had their failureReason attribute populated if the
        upload failed.
        """

        if self.lfn != "" and force == False and lfn == '':
            logger.warning("Warning you're about to 'put' this DiracFile: %s on the grid as it already has an lfn: %s" % (self.namePattern, self.lfn))
            decision = raw_input('y / [n]:')
            while not (decision.lower() in ['y', 'n'] or decision.lower() == ''):
                decision = raw_input('y / [n]:')

            if decision.lower() == 'y':
                pass
            else:
                return

        if (lfn != '' and self.lfn != '') and force == False:
            logger.warning("Warning you're attempting to put this DiracFile: %s" % self.namePattern)
            logger.warning("It currently has an LFN associated with it: %s" % self.lfn)
            logger.warning("Do you want to continue and attempt to upload to: %s" % lfn)
            decision = raw_input('y / [n]:')
            while not (decision.lower() in ['y', 'n', '']):
                decision = raw_input('y / [n]:')

            if decision.lower() == 'y':
                pass
            else:
                return

        if lfn and os.path.basename(lfn) != self.namePattern:
            logger.warning("Changing namePattern from: '%s' to '%s' during put operation" % (self.namePattern, os.path.basename(lfn)))

        if lfn:
            self.lfn = lfn

        # looks like will only need this for the interactive uploading of jobs.
        # Also if any backend need dirac upload on client then when downloaded
        # this will upload then delete the file.

        if self.namePattern == "":
            if self.lfn != '':
                logger.warning("'Put'-ing a file with ONLY an existing LFN makes no sense!")
            raise GangaFileError('Can\'t upload a file without a local file name.')

        sourceDir = self.localDir
        if self.localDir is None:
            sourceDir = os.getcwd()
            # attached to a job, use the joboutputdir
            if self._parent != None and os.path.isdir(self.getJobObject().outputdir):
                sourceDir = self.getJobObject().outputdir

        if not os.path.isdir(sourceDir):
            raise GangaFileError('localDir attribute is not a valid dir, don\'t know from which dir to take the file')

        if regex.search(self.namePattern) is not None:
            if self.lfn != "":
                logger.warning("Cannot specify a single lfn for a wildcard namePattern")
                logger.warning("LFN will be generated automatically")
                self.lfn = ""

        if not self.remoteDir:
            try:
                job = self.getJobObject()
                lfn_folder = os.path.join("GangaJob_%s" % job.getFQID('/'), "OutputFiles")
            except AssertionError:
                t = datetime.datetime.now()
                this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
                lfn_folder = os.path.join('GangaFiles_%s' % this_date)
            lfn_base = os.path.join(DiracFile.diracLFNBase(self.credential_requirements), lfn_folder)

        else:
            lfn_base = os.path.join(DiracFile.diracLFNBase(self.credential_requirements), self.remoteDir)

        if uploadSE == "":
            if self.defaultSE != "":
                storage_elements = [self.defaultSE]
            else:
                if configDirac['allDiracSE']:
                    storage_elements = [random.choice(configDirac['allDiracSE'])]
                else:
                    raise GangaFileError("Can't upload a file without a valid defaultSE or storageSE, please provide one")
        elif isinstance(uploadSE, list):
            storage_elements = uploadSE
        else:
            storage_elements = [uploadSE]

        outputFiles = GangaList()
        for this_file in glob.glob(os.path.join(sourceDir, self.namePattern)):
            name = this_file

            if not os.path.exists(name):
                if not self.compressed:
                    raise GangaFileError('Cannot upload file. File "%s" must exist!' % name)
                name += '.gz'
                if not os.path.exists(name):
                    raise GangaFileError('File "%s" must exist!' % name)
            else:
                if self.compressed:
                    os.system('gzip -c %s > %s.gz' % (name, name))
                    name += '.gz'
                    if not os.path.exists(name):
                        raise GangaFileError('File "%s" must exist!' % name)

            lfn = os.path.join(lfn_base, os.path.basename(this_file))

            d = DiracFile()
            d.namePattern = os.path.basename(name)
            d.compressed = self.compressed
            d.localDir = sourceDir
            stderr = ''
            stdout = ''
            logger.info('Uploading file \'%s\' to \'%s\' as \'%s\'' % (name, storage_elements[0], lfn))
            logger.debug('execute: uploadFile("%s", "%s", %s)' % (lfn, os.path.join(sourceDir, name), str([storage_elements[0]])))
            try:
                stdout = execute('uploadFile("%s", "%s", %s)' % (lfn, os.path.join(sourceDir, name), str([storage_elements[0]])), cred_req=self.credential_requirements)
            except GangaDiracError as err:
                logger.warning("Couldn't upload file '%s': \'%s\'" % (os.path.basename(name), err))
                failureReason = "Error in uploading file '%s' : '%s'" % (os.path.basename(name), err)
                if regex.search(self.namePattern) is not None:
                    d.failureReason = failureReason
                    outputFiles.append(d)
                    continue
                self.failureReason += '\n' + failureReason
                continue

            stdout_temp = stdout.get('Successful')

            if not stdout_temp:
                msg = "Couldn't upload file '%s': \'%s\'" % (os.path.basename(name), stdout)
                logger.warning(msg)
                if regex.search(self.namePattern) is not None:
                    d.failureReason = msg
                    outputFiles.append(d)
                    continue
                self.failureReason = msg
                continue
            else:
                lfn_out = stdout_temp[lfn]

            # when doing the two step upload delete the temp file
            if self.compressed or self._parent != None:
                os.remove(name)
            # need another eval as datetime needs to be included.
            guid = lfn_out.get('GUID', '')
            if regex.search(self.namePattern) is not None:
                d.lfn = lfn
                d.remoteDir = os.path.dirname(lfn)
                d.locations = lfn_out.get('allDiracSE', '')
                d.guid = guid
                outputFiles.append(d)
                continue
            else:
                self.lfn = lfn
                self.remoteDir = os.path.dirname(lfn)
                self.locations = lfn_out.get('allDiracSE', '')
                self.guid = guid

        if replicate == True:

            if len(outputFiles) == 1 or len(outputFiles) == 0:
                storage_elements.pop(0)
                for se in storage_elements:
                    self.replicate(se)
            else:
                storage_elements.pop(0)
                for this_file in outputFiles:
                    for se in storage_elements:
                        this_file.replicate(se)

        if len(outputFiles) > 0:
            return outputFiles
        else:
            outputFiles.append(self)
            return outputFiles
예제 #40
0
class LHCbDataset(GangaDataset):
    '''Class for handling LHCb data sets (i.e. inputdata for LHCb jobs).

    Example Usage:
    ds = LHCbDataset(["lfn:/some/lfn.file","pfn:/some/pfn.file"])
    ds[0] # DiracFile("/some/lfn.file") - see DiracFile docs for usage
    ds[1] # PhysicalFile("/some/pfn.file")- see PhysicalFile docs for usage
    len(ds) # 2 (number of files)
    ds.getReplicas() # returns replicas for *all* files in the data set
    ds.replicate("CERN-USER") # replicate *all* LFNs to "CERN-USER" SE
    ds.getCatalog() # returns XML catalog slice
    ds.optionsString() # returns Gaudi-sytle options 
    [...etc...]
    '''
    schema = {}
    docstr = 'List of PhysicalFile and DiracFile objects'
    schema['files'] = GangaFileItem(
        defvalue=[],
        typelist=['str', 'Ganga.GPIDev.Adapters.IGangaFile.IGangaFile'],
        sequence=1,
        doc=docstr)
    docstr = 'Ancestor depth to be queried from the Bookkeeping'
    schema['depth'] = SimpleItem(defvalue=0, doc=docstr)
    docstr = 'Use contents of file rather than generating catalog.'
    schema['XMLCatalogueSlice'] = GangaFileItem(defvalue=None, doc=docstr)
    docstr = 'Specify the dataset persistency technology'
    schema['persistency'] = SimpleItem(defvalue=None,
                                       typelist=['str', 'type(None)'],
                                       doc=docstr)
    schema['treat_as_inputfiles'] = SimpleItem(
        defvalue=False,
        doc=
        "Treat the inputdata as inputfiles, i.e. copy the inputdata to the WN")

    _schema = Schema(Version(3, 0), schema)
    _category = 'datasets'
    _name = "LHCbDataset"
    _exportmethods = [
        'getReplicas', '__len__', '__getitem__', 'replicate', 'hasLFNs',
        'append', 'extend', 'getCatalog', 'optionsString', 'getLFNs',
        'getFileNames', 'getFullFileNames', 'difference', 'isSubset',
        'isSuperset', 'intersection', 'symmetricDifference', 'union',
        'bkMetadata', 'isEmpty', 'hasPFNs', 'getPFNs'
    ]  # ,'pop']

    def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
        super(LHCbDataset, self).__init__()
        if files is None:
            files = []
        self.files = GangaList()
        process_files = True
        if fromRef:
            self.files._list.extend(files)
            process_files = False
        elif isinstance(files, GangaList):

            def isFileTest(_file):
                return isinstance(_file, IGangaFile)

            areFiles = all([isFileTest(f) for f in files._list])
            if areFiles:
                self.files._list.extend(files._list)
                process_files = False
        elif isinstance(files, LHCbDataset):
            self.files._list.extend(files.files._list)
            process_files = False

        if process_files:
            if isType(files, LHCbDataset):
                for this_file in files:
                    self.files.append(deepcopy(this_file))
            elif isType(files, IGangaFile):
                self.files.append(deepcopy(files))
            elif isType(files, (list, tuple, GangaList)):
                new_list = []
                for this_file in files:
                    if type(this_file) is str:
                        new_file = string_datafile_shortcut_lhcb(
                            this_file, None)
                    elif isType(this_file, IGangaFile):
                        new_file = stripProxy(this_file)
                    else:
                        new_file = strToDataFile(this_file)
                    new_list.append(new_file)
                self.files.extend(new_list)
            elif type(files) is str:
                self.files.append(string_datafile_shortcut_lhcb(files, None),
                                  False)
            else:
                raise GangaException(
                    "Unknown object passed to LHCbDataset constructor!")

        self.files._setParent(self)

        logger.debug("Processed inputs, assigning files")

        # Feel free to turn this on again for debugging but it's potentially quite expensive
        #logger.debug( "Creating dataset with:\n%s" % self.files )

        logger.debug("Assigned files")

        self.persistency = persistency
        self.depth = depth
        logger.debug("Dataset Created")

    def __getitem__(self, i):
        '''Proivdes scripting (e.g. ds[2] returns the 3rd file) '''
        #this_file = self.files[i]
        # print type(this_file)
        # return this_file
        # return this_file
        # return this_file
        if type(i) == type(slice(0)):
            ds = LHCbDataset(files=self.files[i])
            ds.depth = self.depth
            #ds.XMLCatalogueSlice = self.XMLCatalogueSlice
            return ds
        else:
            return self.files[i]

    def getReplicas(self):
        'Returns the replicas for all files in the dataset.'
        lfns = self.getLFNs()
        cmd = 'getReplicas(%s)' % str(lfns)
        result = get_result(cmd, 'LFC query error. Could not get replicas.')
        return result['Successful']

    def hasLFNs(self):
        'Returns True is the dataset has LFNs and False otherwise.'
        for f in self.files:
            if isDiracFile(f):
                return True
        return False

    def hasPFNs(self):
        'Returns True is the dataset has PFNs and False otherwise.'
        for f in self.files:
            if not isDiracFile(f):
                return True
        return False

    def replicate(self, destSE=''):
        '''Replicate all LFNs to destSE.  For a list of valid SE\'s, type
        ds.replicate().'''

        if not destSE:
            from GangaDirac.Lib.Files.DiracFile import DiracFile
            DiracFile().replicate('')
            return
        if not self.hasLFNs():
            raise GangaException('Cannot replicate dataset w/ no LFNs.')

        retry_files = []

        for f in self.files:
            if not isDiracFile(f):
                continue
            try:
                result = f.replicate(destSE=destSE)
            except Exception as err:
                msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
                logger.warning(msg)
                logger.warning("Error: %s" % str(err))
                retry_files.append(f)

        for f in retry_files:
            try:
                result = f.replicate(destSE=destSE)
            except Exception as err:
                msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
                logger.warning(msg)
                logger.warning(str(err))

    def extend(self, files, unique=False):
        '''Extend the dataset. If unique, then only add files which are not
        already in the dataset.'''
        from Ganga.GPIDev.Base import ReadOnlyObjectError

        if self._parent is not None and self._parent._readonly():
            raise ReadOnlyObjectError(
                'object Job#%s  is read-only and attribute "%s/inputdata" cannot be modified now'
                % (self._parent.id, getName(self)))

        _external_files = []

        if type(files) is str or isType(files, IGangaFile):
            _external_files = [files]
        elif type(files) in [list, tuple]:
            _external_files = files
        elif isType(files, LHCbDataset):
            _external_files = files.files
        else:
            if not hasattr(files, "__getitem__") or not hasattr(
                    files, '__iter__'):
                _external_files = [files]

        # just in case they extend w/ self
        _to_remove = []
        for this_file in _external_files:
            if hasattr(this_file, 'subfiles'):
                if len(this_file.subfiles) > 0:
                    _external_files = makeGangaListByRef(this_file.subfiles)
                    _to_remove.append(this_file)
            if type(this_file) is str:
                _external_files.append(
                    string_datafile_shortcut_lhcb(this_file, None))
                _to_remove.append(this_file)

        for _this_file in _to_remove:
            _external_files.pop(_external_files.index(_this_file))

        for this_f in _external_files:
            _file = getDataFile(this_f)
            if _file is None:
                _file = this_f
            myName = _file.namePattern
            from GangaDirac.Lib.Files.DiracFile import DiracFile
            if isType(_file, DiracFile):
                myName = _file.lfn
            if unique and myName in self.getFileNames():
                continue
            self.files.append(stripProxy(_file))

    def removeFile(self, input_file):
        try:
            self.files.remove(input_file)
        except:
            raise GangaException('Dataset has no file named %s' %
                                 input_file.namePattern)

    def getLFNs(self):
        'Returns a list of all LFNs (by name) stored in the dataset.'
        lfns = []
        if not self:
            return lfns
        for f in self.files:
            if isDiracFile(f):
                subfiles = f.getSubFiles()
                if len(subfiles) == 0:
                    lfns.append(f.lfn)
                else:
                    for file in subfiles:
                        lfns.append(file.lfn)

        #logger.debug( "Returning LFNS:\n%s" % str(lfns) )
        logger.debug("Returning #%s LFNS" % str(len(lfns)))
        return lfns

    def getPFNs(self):
        'Returns a list of all PFNs (by name) stored in the dataset.'
        pfns = []
        if not self:
            return pfns
        for f in self.files:
            if isPFN(f):
                pfns.append(f.namePattern)
        return pfns

    def getFullFileNames(self):
        'Returns all file names w/ PFN or LFN prepended.'
        names = []
        from GangaDirac.Lib.Files.DiracFile import DiracFile
        for f in self.files:
            if isType(f, DiracFile):
                names.append('LFN:%s' % f.lfn)
            else:
                try:
                    names.append('PFN:%s' % f.namePattern)
                except:
                    logger.warning("Cannot determine filename for: %s " % f)
                    raise GangaException("Cannot Get File Name")
        return names

    def getCatalog(self, site=''):
        '''Generates an XML catalog from the dataset (returns the XML string).
        Note: site defaults to config.LHCb.LocalSite
        Note: If the XMLCatalogueSlice attribute is set, then it returns
              what is written there.'''
        if hasattr(self.XMLCatalogueSlice, 'name'):
            if self.XMLCatalogueSlice.name:
                f = open(self.XMLCatalogueSlice.name)
                xml_catalog = f.read()
                f.close()
                return xml_catalog
        if not site:
            site = getConfig('LHCb')['LocalSite']
        lfns = self.getLFNs()
        depth = self.depth
        tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml')
        cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \
              % (str(lfns), depth, site, tmp_xml.name)
        result = get_result(cmd, 'LFN->PFN error. XML catalog error.')
        xml_catalog = tmp_xml.read()
        tmp_xml.close()
        return xml_catalog

    def optionsString(self, file=None):
        'Returns the Gaudi-style options string for the dataset (if a filename' \
            ' is given, the file is created and output is written there).'
        if not self or len(self) == 0:
            return ''
        snew = ''
        if self.persistency == 'ROOT':
            snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency
        elif self.persistency == 'POOL':
            snew = '\ntry:\n    #new method\n    from GaudiConf import IOExtension\n    IOExtension(\"%s\").inputFiles([' % self.persistency
        elif self.persistency == None:
            snew = '\ntry:\n    #new method\n    from GaudiConf import IOExtension\n    IOExtension().inputFiles(['
        else:
            logger.warning(
                "Unknown LHCbDataset persistency technology... reverting to None"
            )
            snew = '\ntry:\n    #new method\n    from GaudiConf import IOExtension\n    IOExtension().inputFiles(['

        sold = '\nexcept ImportError:\n    #Use previous method\n    from Gaudi.Configuration import EventSelector\n    EventSelector().Input=['
        sdatasetsnew = ''
        sdatasetsold = ''

        dtype_str_default = getConfig('LHCb')['datatype_string_default']
        dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns']
        for f in self.files:
            dtype_str = dtype_str_default
            for this_str in dtype_str_patterns:
                matched = False
                for pat in dtype_str_patterns[this_str]:
                    if fnmatch.fnmatch(f.namePattern, pat):
                        dtype_str = this_str
                        matched = True
                        break
                if matched:
                    break
            sdatasetsnew += '\n        '
            sdatasetsold += '\n        '
            if isDiracFile(f):
                sdatasetsnew += """ \"LFN:%s\",""" % f.lfn
                sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn,
                                                                    dtype_str)
            else:
                sdatasetsnew += """ \"%s\",""" % f.accessURL()[0]
                sdatasetsold += """ \"DATAFILE='%s' %s\",""" % (
                    f.accessURL()[0], dtype_str)
        if sdatasetsold.endswith(","):
            if self.persistency == 'ROOT':
                sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)"""
            else:
                sdatasetsnew = sdatasetsnew[:-1] + """\n    ], clear=True)"""
            sdatasetsold = sdatasetsold[:-1]
            sdatasetsold += """\n    ]"""
        if (file):
            f = open(file, 'w')
            if self.persistency == 'ROOT':
                f.write(snew)
                f.write(sdatasetsnew)
            else:
                f.write(snew)
                f.write(sdatasetsnew)
                f.write(sold)
                f.write(sdatasetsold)
            f.close()
        else:
            if self.persistency == 'ROOT':
                return snew + sdatasetsnew
            else:
                return snew + sdatasetsnew + sold + sdatasetsold

    def _checkOtherFiles(self, other):
        if isType(other, GangaList) or isType(other, []):
            other_files = LHCbDataset(other).getFullFileNames()
        elif isType(other, LHCbDataset):
            other_files = other.getFullFileNames()
        else:
            raise GangaException("Unknown type for difference")
        return other_files

    def difference(self, other):
        '''Returns a new data set w/ files in this that are not in other.'''
        other_files = self._checkOtherFiles(other)
        files = set(self.getFullFileNames()).difference(other_files)
        data = LHCbDataset()
        data.extend(list(files))
        data.depth = self.depth
        return data

    def isSubset(self, other):
        '''Is every file in this data set in other?'''
        other_files = self._checkOtherFiles(other)
        return set(self.getFileNames()).issubset(other_files)

    def isSuperset(self, other):
        '''Is every file in other in this data set?'''
        other_files = self._checkOtherFiles(other)
        return set(self.getFileNames()).issuperset(other_files)

    def symmetricDifference(self, other):
        '''Returns a new data set w/ files in either this or other but not
        both.'''
        other_files = other._checkOtherFiles(other)
        files = set(self.getFullFileNames()).symmetric_difference(other_files)
        data = LHCbDataset()
        data.extend(list(files))
        data.depth = self.depth
        return data

    def intersection(self, other):
        '''Returns a new data set w/ files common to this and other.'''
        other_files = other._checkOtherFiles(other)
        files = set(self.getFullFileNames()).intersection(other_files)
        data = LHCbDataset()
        data.extend(list(files))
        data.depth = self.depth
        return data

    def union(self, other):
        '''Returns a new data set w/ files from this and other.'''
        other_files = self._checkOtherFiles(other)
        files = set(self.getFullFileNames()).union(other_files)
        data = LHCbDataset()
        data.extend(list(files))
        data.depth = self.depth
        return data

    def bkMetadata(self):
        'Returns the bookkeeping metadata for all LFNs. '
        logger.info(
            "Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info..."
        )
        cmd = 'bkMetaData(%s)' % self.getLFNs()
        b = get_result(cmd, 'Error removing replica. Replica rm error.')
        return b
예제 #41
0
    def put(self, lfn='', force=False, uploadSE="", replicate=False):
        """
        Try to upload file sequentially to storage elements defined in configDirac['allDiracSE'].
        File will be uploaded to the first SE that the upload command succeeds for.

        The file is uploaded to the SE described by the DiracFile.defaultSE attribute

        Alternatively, the user can specify an uploadSE which contains an SE
        which the file is to be uploaded to.

        If the user wants to replicate this file(s) across all SE then they should state replicate = True.

        Return value will be either the stdout from the dirac upload command if not
        using the wildcard characters '*?[]' in the namePattern.
        If the wildcard characters are used then the return value will be a list containing
        newly created DiracFile objects which were the result of glob-ing the wildcards.

        The objects in this list will have been uploaded or had their failureReason attribute populated if the
        upload failed.
        """

        if self.lfn != "" and force == False and lfn == '':
            logger.warning("Warning you're about to 'put' this DiracFile: %s on the grid as it already has an lfn: %s" % (self.namePattern, self.lfn))
            decision = raw_input('y / [n]:')
            while not (decision in ['y', 'n'] or decision == ''):
                decision = raw_input('y / [n]:')

            if decision == 'y':
                pass
            else:
                return

        if (lfn != '' and self.lfn != '') and force == False:
            logger.warning("Warning you're attempting to put this DiracFile: %s" % self.namePattern)
            logger.warning("It currently has an LFN associated with it: %s" % self.lfn)
            logger.warning("Do you want to continue and attempt to upload to: %s" % lfn)
            decision = raw_input('y / [n]:')
            while not (decision in ['y', 'n', '']):
                decision = raw_input('y / [n]:')

            if decision == 'y':
                pass
            else:
                return

        if lfn != '':
            self.lfn = lfn

        # looks like will only need this for the interactive uploading of jobs.
        # Also if any backend need dirac upload on client then when downloaded
        # this will upload then delete the file.

        if self.namePattern == "":
            if self.lfn != '':
                logger.warning("'Put'-ing a file with ONLY an existing LFN makes no sense!")
            raise GangaException('Can\'t upload a file without a local file name.')

        sourceDir = self.localDir
        if self.localDir is None:
            sourceDir = os.getcwd()
            # attached to a job, use the joboutputdir
            if self._parent != None and os.path.isdir(self.getJobObject().outputdir):
                sourceDir = self.getJobObject().outputdir

        if not os.path.isdir(sourceDir):
            raise GangaException('localDir attribute is not a valid dir, don\'t know from which dir to take the file')

        if regex.search(self.namePattern) is not None:
            if self.lfn != "":
                logger.warning("Cannot specify a single lfn for a wildcard namePattern")
                logger.warning("LFN will be generated automatically")
                self.lfn = ""

        selfConstructedLFN = False

        import glob
        if self.remoteDir == '' and self.lfn == '':
            import datetime
            t = datetime.datetime.now()
            this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
            self.lfn = os.path.join(configDirac['DiracLFNBase'], 'GangaFiles_%s' % this_date)
            selfConstructedLFN = True
        #if self.remoteDir == '' and self.lfn != '':
        #    self.remoteDir = configDirac['DiracLFNBase']

        if self.remoteDir[:4] == 'LFN:':
            lfn_base = self.remoteDir[4:]
        else:
            lfn_base = self.remoteDir

        if uploadSE != "":
            storage_elements = all_SE_list(uploadSE)
        else:
            storage_elements = all_SE_list(self.defaultSE)

        outputFiles = GangaList()
        for this_file in glob.glob(os.path.join(sourceDir, self.namePattern)):
            name = this_file

            if not os.path.exists(name):
                if not self.compressed:
                    raise GangaException('Cannot upload file. File "%s" must exist!' % name)
                name += '.gz'
                if not os.path.exists(name):
                    raise GangaException('File "%s" must exist!' % name)
            else:
                if self.compressed:
                    os.system('gzip -c %s > %s.gz' % (name, name))
                    name += '.gz'
                    if not os.path.exists(name):
                        raise GangaException('File "%s" must exist!' % name)

            if lfn == "":
                lfn = os.path.join(lfn_base, os.path.basename(name))

            if selfConstructedLFN is True:
                self.lfn = os.path.join(self.lfn, os.path.basename(name))

            lfn = self.lfn

            d = DiracFile()
            d.namePattern = os.path.basename(name)
            d.compressed = self.compressed
            d.localDir = sourceDir
            stderr = ''
            stdout = ''
            logger.info('Uploading file %s to %s as %s' % (name, storage_elements[0], lfn))
            stdout = execute('uploadFile("%s", "%s", %s)' % (lfn, name, str([storage_elements[0]])))
            if type(stdout) == str:
                logger.warning("Couldn't upload file '%s': %s" % (os.path.basename(name), stdout))
                continue
            if stdout.get('OK', False) and lfn in stdout.get('Value', {'Successful': {}})['Successful']:
                # when doing the two step upload delete the temp file
                if self.compressed or self._parent != None:
                    os.remove(name)
                # need another eval as datetime needs to be included.
                guid = stdout['Value']['Successful'][lfn].get('GUID', '')
                if regex.search(self.namePattern) is not None:
                    d.lfn = lfn
                    d.remoteDir = os.path.dirname(lfn)
                    d.locations = stdout['Value']['Successful'][lfn].get('allDiracSE', '')
                    d.guid = guid
                    outputFiles.append(GPIProxyObjectFactory(d))
                    continue
                else:
                    self.lfn = lfn
                    self.remoteDir = os.path.dirname(lfn)
                    self.locations = stdout['Value']['Successful'][lfn].get('allDiracSE', '')
                    self.guid = guid
                # return ## WHY?
            else:
                failureReason = "Error in uploading file %s : %s" % (os.path.basename(name), str(stdout))
                logger.error(failureReason)
                if regex.search(self.namePattern) is not None:
                    d.failureReason = failureReason
                    outputFiles.append(GPIProxyObjectFactory(d))
                    continue
                self.failureReason = failureReason
                return str(stdout)

        if replicate == True:

            if len(outputFiles) == 1 or len(outputFiles) == 0:
                storage_elements.pop(0)
                for se in storage_elements:
                    self.replicate(se)
            else:
                storage_elements.pop(0)
                for this_file in outputFiles:
                    for se in storage_elements:
                        this_file.replicate(se)

        if len(outputFiles) > 0:
            return GPIProxyObjectFactory(outputFiles)
        else:
            outputFiles.append(self)
            return GPIProxyObjectFactory(outputFiles)
예제 #42
0
class TestGangaList(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        super(TestGangaList, self).__init__(*args, **kwargs)

        self.plain1 = []
        self.plain2 = []

        self.proxied1 = []
        self.proxied2 = []

        self.ganga_list = GangaList()

    @staticmethod
    def _makeRandomString():
        str_len = random.randint(3, 10)
        s = ''
        for _ in range(str_len):
            s += random.choice(string.ascii_letters)
        return s

    @staticmethod
    def _makeRandomTFile():
        name = TestGangaList._makeRandomString()
        subdir = TestGangaList._makeRandomString()
        return TFile(name=name, subdir=subdir)

    def setUp(self):
        super(TestGangaList, self).setUp()

        self.plain1 = [self._makeRandomTFile() for _ in range(15)]
        self.plain2 = [self._makeRandomTFile() for _ in range(10)]

        self.proxied1 = GangaList()
        self.proxied1.extend(self.plain1[:])
        self.proxied2 = GangaList()
        self.proxied2.extend(self.plain2[:])

        t = TFile()
        real_t = stripProxy(t)
        new_proxy_t = addProxy(real_t)
        #hopefully_t = stripProxy(new_proxy_t)
        #assert real_t is hopefully_t
        assert t is new_proxy_t

        self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction")
        self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction")

    def testAllListMethodsExported(self):
        """Tests that all methods on list are exposed by GangaList"""

        def getmethods(_obj):
            # get all the method names for an object
            return dir(_obj)

        list_methods = getmethods([])
        gangalist_methods = getmethods(self.ganga_list)

        missing_methods = []
        for m in list_methods:
            if not m in gangalist_methods:
                missing_methods.append(m)

        if missing_methods:
            logger.info(missing_methods)

        self.assertFalse(missing_methods, 'Not all of lists methods are implemented: %s' % missing_methods)

    def testEq(self):
        """Tests the equality op that the rest of the tests rely on."""
        self.assertEqual(self.proxied1, self.plain1, 'Proxied and non-proxied versions should be the same')

    def testNq(self):
        self.assertIsNotNone(self.proxied1, None)
        self.assertFalse(self.proxied1 != self.proxied1, 'Lists are the same')
        self.assertNotEqual(self.proxied1, self.proxied2, 'Lists are different')
        self.assertNotEqual(self.plain1, self.proxied2, 'Lists are different')

    def testNonZero(self):
        """@ExpectedFailure"""
        self.assertFalse(GangaList(), 'An empty GangaList should be false, just like a list')

    def testAdd(self):
        """Test __add__"""
        self.assertEqual(self.plain1 + self.plain2, self.proxied1 + self.proxied2)
        self.assertTrue(isProxy(self.proxied1 + self.proxied2))

    def testAddMixed(self):
        """Test __add__ with mixed lists and GangaLists"""
        self.assertEqual((self.plain1 + self.plain2), (self.proxied1 + self.plain2))
        self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1))
        self.assertTrue(isProxy(self.proxied1 + self.plain2))

        self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1))
        self.assertEqual((self.plain1 + self.plain2), (self.plain1 + self.proxied2))
        self.assertTrue(isinstance(self.plain1 + self.proxied2, list))

    def testAddMixed2(self):

        self.plain1 = range(10)
        self.plain2 = range(10)

        self.assertTrue(isProxy(self.proxied2[-1]), 'Element access must get proxies')
        self.assertFalse(isProxy(self.plain1[0]), 'Element access must not proxies')
        self.assertTrue(isProxy((self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies')
        self.assertFalse(isProxy((self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone')

        self.assertEqual((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be equal')
        self.assertIs((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be identical')

    def testAddStr(self):
        """Makes sure that only lists can be added."""
        try:
            [] + ''
            assert False, 'Line above should throw a TypeError'
        except TypeError:
            pass

        try:
            self.proxied1 + ''
            assert False, 'Line above should throw a TypeError'
        except TypeError:
            pass

    def testContains(self):
        """Tests __contains__"""

        plist = [addProxy(x) for x in self.plain1]
        self.assertEqual(plist, self.proxied1)

        for p in plist:
            self.assertTrue(isProxy(p))
            self.assertIn(p, self.proxied1, 'Proxied list should contain each proxied object')

    def testDelItem(self):
        """Test __delitem__"""

        for p in [addProxy(x) for x in self.plain1[:]]:
            self.assertTrue(isProxy(p))
            del self.proxied1[self.proxied1.index(p)]

    def testGE(self):
        """Test __ge__"""

        self.assertEqual((self.plain1 >= self.plain2), (self.proxied1 >= self.proxied2), 'The lists should have the same ge')
        self.assertEqual((self.plain2 >= self.plain1), (self.proxied2 >= self.proxied1), 'The lists should have the same ge')

        self.assertNotEqual((self.proxied1 >= self.proxied2), (self.proxied2 >= self.proxied1), 'The gt should invert correctly')

    def testGetItem(self):
        """Test __getitem__"""

        for i in range(len(self.proxied1)):
            self.assertTrue(isProxy(self.proxied1[i]))

    def testGetSlice(self):
        """Test __getslice__"""

        slices = [(0, 0), (0, len(self.plain1))]

        for s in slices:
            self.assertEqual(self.plain1[s[0]:s[1]], self.proxied1[s[0]:s[1]], 'Slices {0} should be the same'.format(s))

        t = self.plain1[:]
        self.assertIsNot(t, self.plain1, 'Slice should be a copy.')
        self.assertIsNot(self.plain1[:], t)

        t = self.proxied1[:]
        self.assertIsNot(t, self.proxied1, 'Slice should be a copy.')
        self.assertIsNot(self.proxied1[:], t)

    def testGT(self):
        """Test __gt__"""

        self.assertEqual((self.plain1 > self.plain2), (self.proxied1 > self.proxied2), 'The lists should have the same gt')
        self.assertEqual((self.plain2 > self.plain1), (self.proxied2 > self.proxied1), 'The lists should have the same gt')

        self.assertNotEqual((self.proxied1 > self.proxied2), (self.proxied2 > self.proxied1), 'The gt should invert correctly')

    def testIAdd(self):
        """Test __iadd__"""
        self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
        self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')

        self.plain1 += self.plain2
        self.proxied1 += self.proxied2

        self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same')
        self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')

    def testIAddMixed(self):
        """Test __iadd__ where we mix lists and GangaLists"""
        self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
        self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')

        self.plain1 += self.plain2
        self.proxied1 += self.plain2

        self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same')
        self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
        self.assertTrue(isinstance(self.plain1, list), 'Must be list instance')

        self.plain2 += self.proxied1
        self.proxied2 += self.proxied1

        self.assertEqual(self.plain2, self.proxied2, 'Addition should be the same')
        self.assertTrue(isProxy(self.proxied2), 'Proxy must be added')
        self.assertTrue(isinstance(self.plain2, list), 'Must be list instance')

    def testIAddSlice(self):
        """Test __iadd__ on slices"""

        s1 = self.plain1[3:7]
        s1 += self.plain2[2:5]

        s2 = self.proxied1[3:7]
        s2 += self.proxied2[2:5]

        self.assertEqual(s1, s2, 'Addition should be the same')

        self.assertFalse(isProxy(s1), 'Proxy Not Needed')
        self.assertTrue(isProxy(s2), 'Proxy Needed')

    def testIdentity(self):
        """Tests we obey list like identity relations."""

        t = self.plain1[4]
        self.assertIs(self.plain1[4], t)

        t = self.proxied1[4]
        self.assertIs(self.proxied1[4], t)

    def testIMul(self):
        """Test __imul__"""

        self.plain1 *= 5
        self.proxied1 *= 5

        self.assertEqual(self.plain1, self.proxied1, 'Multiplication should be the same')
        self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')

    def testIMulSlice(self):
        """Test __imul__ on slices"""

        s1 = self.plain1[3:7]
        s1 *= 6

        s2 = self.proxied1[3:7]
        s2 *= 6

        self.assertEqual(s1, s2, 'Addition should be the same')

    def testLE(self):
        """Test __le__"""

        self.assertEqual(self.plain1 <= self.plain2, self.proxied1 <= self.proxied2, 'The lists should have the same le')
        self.assertEqual(self.plain2 <= self.plain1, self.proxied2 <= self.proxied1, 'The lists should have the same le')

        self.assertNotEqual(self.proxied1 <= self.proxied2, self.proxied2 <= self.proxied1, 'The le should invert correctly')

    def testLen(self):
        """Tests __len__"""
        self.assertEqual(len(self.plain1), len(self.proxied1), 'Lengths should be the same')

    def testLT(self):
        """Test __lt__"""

        self.assertEqual(self.plain1 < self.plain2, self.proxied1 < self.proxied2, 'The lists should have the same lt')
        self.assertEqual(self.plain2 < self.plain1, self.proxied2 < self.proxied1, 'The lists should have the same lt')

        self.assertNotEqual(self.proxied1 < self.proxied2, self.proxied2 < self.proxied1, 'The lt should invert correctly')

    def testMul(self):
        """Test __mul__"""
        self.assertEqual((self.plain1 * 7), (self.proxied1 * 7))
        self.assertTrue(isProxy(self.proxied1 * 9))

        for p in self.proxied1:
            self.assertTrue(isProxy(p))

    def testNE(self):
        """Test __ne__"""

        self.assertNotEqual(self.plain1, self.plain2)
        self.assertNotEqual(self.proxied1, self.proxied2, 'Lists should be different')

        self.assertNotEqual(self.plain1[0:5], self.plain1[2:7])
        self.assertNotEqual(self.proxied1[0:5], self.proxied1[2:7], 'Lists should be different')

    def testRMul(self):
        """Test __rmul__"""

        t1 = 5 * self.plain1
        t2 = 5 * self.proxied1

        self.assertEqual(t1, t2, 'Multiplication should be the same')

    def testReversed(self):
        """Test the __reversed__ feature (new in python 2.4)."""

        count = len(self.proxied1) - 1
        for i in self.proxied1.__reversed__():
            self.assertIs(i, self.proxied1[count])
            self.assertTrue(isProxy(i))
            self.assertTrue(isType(i, TFile))
            count -= 1

    def testSetItem(self):
        """Test __setitem__"""

        t = TFile(name='foo', subdir='cheese')
        test_index = 7
        self.plain1[test_index] = t
        self.assertIs(self.plain1[test_index], t)

        self.proxied1[test_index] = t
        self.assertIs(self.proxied1[test_index], t)
        self.assertTrue(isProxy(self.proxied1[test_index]))

    def testSetSliceProxyList(self):

        self.plain1[3:7] = self.plain2[3:7]
        self.assertEqual(self.plain1[3:7], self.plain2[3:7], 'The lists should be equal')
        self.assertIsNot(self.plain1[3:7], self.plain2[3:7], 'The lists should be copies')
        self.plain1[4] = self.plain2[9]
        self.assertNotEqual(self.plain1[4], self.plain2[4])

        tmp = self.plain1[2:9]
        self.assertIsNot(self.plain1[2:9], tmp)

        self.proxied1[3:7] = self.proxied2[3:7]
        self.assertEqual(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be equal')
        self.assertIsNot(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be copies')
        self.proxied1[4] = self.proxied2[9]
        self.assertNotEqual(self.proxied1[4], self.proxied2[4])

        tmp = self.proxied1[2:9]
        self.assertIsNot(self.proxied1[2:9], tmp)

    def testAppend(self):

        t = TFile(name='foo')

        self.plain1.append(t)
        self.assertEqual(self.plain1[-1], t)
        self.assertEqual(self.plain1.pop(), t)

        self.proxied1.append(t)
        self.assertEqual(self.proxied1[-1], t)
        self.assertIs(self.proxied1[-1], t, 'Identity Test')
        self.assertTrue(isProxy(self.proxied1[-1]), 'Make sure we get back a proxy')
        self.assertEqual(self.proxied1.pop(), t)

    def testExtend(self):

        t1 = [self._makeRandomTFile() for _ in xrange(10)]

        self.plain1.extend(t1)
        self.proxied1.extend(t1)

        self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same')

        t2 = self.proxied1[4:7]
        self.assertTrue(isProxy(t2))
        from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
        self.assertTrue(isType(t2, glist))
        self.plain1.extend(t2)
        self.proxied1.extend(t2)
        self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same')

    def testIndex(self):

        t = TFile(name='foo')
        self.proxied1.insert(8, t)
        self.assertEqual(self.proxied1[8], t)
        self.assertEqual(self.proxied1.index(t), 8)

    def testInsert(self):

        t = TFile(name='foo')
        self.proxied1.insert(8, t)
        self.assertEqual(self.proxied1[8], t)

    def testPop(self):

        list_len = len(self.proxied1)

        t = self.proxied1[-1]
        r = self.proxied1.pop()
        self.assertEqual(t, r)
        self.assertIs(t, r)
        self.assertEqual(len(self.proxied1), list_len - 1)
        self.assertNotIn(t, self.proxied1)
        self.assertNotIn(t._impl, self.proxied1._impl)

        t = self.proxied1[6]
        r = self.proxied1.pop(6)
        self.assertEqual(t, r)
        self.assertIs(t, r)
        self.assertEqual(len(self.proxied1), list_len - 2)
        self.assertNotIn(t, self.proxied1)
        self.assertNotIn(t._impl, self.proxied1._impl)

    def testRemove(self):

        t = TFile(name='bar')
        self.proxied1.insert(7, t)
        list_len = len(self.proxied1)

        self.proxied1.remove(t)

        self.assertEqual(len(self.proxied1), list_len - 1)
        self.assertNotIn(t, self.proxied1)
        self.assertNotIn(t._impl, self.proxied1._impl)

    def testIter(self):

        count = 0
        for f in self.proxied1:
            count += 1
            self.assertTrue(isProxy(f))
            self.assertTrue(isType(f, TFile))

        self.assertEqual(count, len(self.proxied1), 'Must visit every member')

    def testCmp(self):

        self.assertEqual(cmp(self.proxied1, self.proxied2), cmp(self.plain1, self.plain2))

    def testHash(self):

        try:
            hash(int(self.proxied1))
            assert False, 'Lists are not hashable'
        except TypeError:
            pass