예제 #1
0
    def lookup(self, sharedir, unprepare=False):
        """
        Report Job, Box and Task repository items which reference a given ShareDir object. 
        The optional parameter 'unprepare=True' can be set to call the unprepare method 
        on the returned objects.
        """
        from Ganga.GPI import jobs, box, tasks
        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing: 'job'})
        for thing in box.select():
            objectlist.append({thing: 'box'})
        for thing in tasks.select():
            objectlist.append({thing: 'task'})

        run_unp = None
        master_index = 0
        for item in objectlist:
            try:
                item.keys()[0].is_prepared.name
                if item.keys()[0].is_prepared.name == sharedir:
                    logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
                                stripProxy(item.keys()[0])._registry_id, item.values()[0])
                    run_unp = item.keys()[0]
                    master_index += 1
            except AttributeError as err:
                logger.debug("Err: %s" % str(err))
                try:
                    item.keys()[0].application.is_prepared.name
                    if item.keys()[0].application.is_prepared.name == sharedir:
                        logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
                                    stripProxy(item.keys()[0])._registry_id, item.values()[0])
                        run_unp = item.keys()[0].application
                        master_index += 1
                except AttributeError as err2:
                    logger.debug("Err2: %s" % str(err2))
                    try:
                        item.keys()[0].analysis.application.is_prepared.name
                        if item.keys()[0].analysis.application.is_prepared.name == sharedir:
                            logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
                                        stripProxy(item.keys()[0])._registry_id, item.values()[0])
                            run_unp = item.keys()[0].analysis.application
                            master_index += 1
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % str(err3))
                        pass

            if run_unp is not None and unprepare is True:
                logger.info('Unpreparing %s repository object #%s associated with ShareDir %s',
                            item.values()[0],  stripProxy(item.keys()[0])._registry_id, sharedir)
#                stripProxy(item.keys()[0]).unprepare()
                run_unp.unprepare()
                run_unp = None

        if unprepare is not True:
            logger.info('%s item(s) found referencing ShareDir %s', master_index, sharedir)
예제 #2
0
    def rebuild(self, unprepare=True, rmdir=False):
        """Rebuild the shareref table. 
        Clears the shareref table and then rebuilds it by iterating over all Ganga Objects 
        in the Job, Box and Task repositories. If an object has a ShareDir associated with it, 
        that ShareDir is added into the shareref table (the reference counter being incremented 
        accordingly). If called with the optional parameter 'unprepare=False', objects whose
        ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
        is unprepare=True, i.e. the job/application would be unprepared.
        After all Job/Box/Task objects have been checked, the inverse operation is performed, 
        i.e., for each directory in the ShareDir repository, a check is done to ensure there 
        is a matching entry in the shareref table. If not, and the optional parameter 
        'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem. 
        Otherwise, it will be added to the shareref table with a reference count of zero; 
        this results in the directory being deleted upon Ganga exit.
        """
        self._getWriteAccess()
        from Ganga.GPI import jobs, box, tasks
        # clear the shareref table
        self.name = {}
        lookup_input = []

        from Ganga.GPIDev.Lib.File import getSharedPath


        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing: 'job'})
        for thing in box.select():
            objectlist.append({thing: 'box'})
        for thing in tasks.select():
            objectlist.append({thing: 'task'})

        for item in objectlist:
            shortname = None
            try:
                shortname = item.keys()[0].is_prepared.name
            except AttributeError as err:
                logger.debug("Err: %s" % str(err))
                try:
                    shortname = item.keys()[0].application.is_prepared.name
                except AttributeError as err2:
                    logger.debug("Err2: %s" % str(err2))
                    try:
                        shortname = item.keys()[0].analysis.application.is_prepared.name
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % str(err3))
                        pass
            try:
                if shortname is not None and shortname is not True:
                    if os.path.basename(shortname) != shortname:
                        self.to_relative(item.keys()[0].is_prepared)
                    try:
                        numsubjobs = len(item.keys()[0].subjobs.ids())
                    except Exception as err:
                        logger.debug("Path Error: %s" % str(err))
                        Ganga.Utility.logging.log_unknown_exception()
                        numsubjobs = 0
                    self.helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
            except Exception as err:
                logger.debug("-Error: %s" % str(err))
                Ganga.Utility.logging.log_unknown_exception()
                pass

        # here we iterate over the lookup_input list and unprepare as
        # necessary.
        for item in lookup_input:
            logger.info('Unpreparing objects referencing ShareDir %s' % item)
            self.lookup(sharedir=item, unprepare=True)

        # check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
        # so the user is made aware of them at shutdown
        for this_dir in os.listdir(getSharedPath()):
            if this_dir not in self.__getName().keys() and rmdir is False:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % this_dir)
                self.__getName()[this_dir] = 0
            elif this_dir not in self.__getName().keys() and rmdir is True:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % this_dir)
                shutil.rmtree(os.path.join(getSharedPath(), this_dir))

        self._setDirty()
        self._releaseWriteAccess()
예제 #3
0
    def rebuild(self, unprepare=True, rmdir=False):
        """Rebuild the shareref table. 
        Clears the shareref table and then rebuilds it by iterating over all Ganga Objects 
        in the Job, Box and Task repositories. If an object has a ShareDir associated with it, 
        that ShareDir is added into the shareref table (the reference counter being incremented 
        accordingly). If called with the optional parameter 'unprepare=False', objects whose
        ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
        is unprepare=True, i.e. the job/application would be unprepared.
        After all Job/Box/Task objects have been checked, the inverse operation is performed, 
        i.e., for each directory in the ShareDir repository, a check is done to ensure there 
        is a matching entry in the shareref table. If not, and the optional parameter 
        'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem. 
        Otherwise, it will be added to the shareref table with a reference count of zero; 
        this results in the directory being deleted upon Ganga exit.
        """
        self._getWriteAccess()
        from Ganga.GPI import jobs, box, tasks
        #clear the shareref table
        self.name={}
        lookup_input=[]


        def helper(object, unp=True, numsubjobs=0):
            shareddir =  os.path.join(ShareDir._root_shared_path,os.path.basename(object))
            logger.debug('Adding %s to the shareref table.' % shareddir)
            if self.name.has_key(os.path.basename(object)):
                self.name[os.path.basename(object)] +=1
            else:
                self.name[os.path.basename(object)] = 1
            if numsubjobs > 0:
                self.name[os.path.basename(object)] += numsubjobs
            if not os.path.isdir(shareddir) and os.path.basename(object) not in lookup_input:
                logger.info('Shared directory %s not found on disk.' % shareddir)
                if unp == True: 
                    lookup_input.append(os.path.basename(object))


        def to_relative(object):
            logger.info('Absolute ShareDir().name attribute found in Job #%s', object.id)
            logger.info('Converting to relative path and moving associated directory if it exists.')
            try:
                shutil.move(object.is_prepared.name,\
                    os.path.join(ShareDir._root_shared_path,os.path.basename(object.is_prepared.name)))
            except:
                logger.warn('Unable to move directory %s to %s', object.is_prepared.name,\
                    os.path.join(ShareDir._root_shared_path,os.path.basename(object.is_prepared.name)))
            try:
                object._impl.is_prepared.name = os.path.basename(object.is_prepared.name)
            except:
                logger.warn("Unable to convert object's is_prepared.name attribute to a relative path")
        

        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing:'job'})
        for thing in box.select():
            objectlist.append({thing:'box'})
        for thing in tasks.select():
            objectlist.append({thing:'task'})

        for item in objectlist:
            shortname = None
            try:
                shortname = item.keys()[0].is_prepared.name 
            except AttributeError:
                try:
                    shortname = item.keys()[0].application.is_prepared.name 
                except AttributeError:
                    try:
                        shortname = item.keys()[0].analysis.application.is_prepared.name 
                    except AttributeError:
                        pass
            try:
                if shortname is not None and shortname is not True:
                    if os.path.basename(shortname) != shortname:
                        to_relative(item.keys()[0].is_prepared)
                    try:
                        numsubjobs = len(item.keys()[0].subjobs.ids())
                    except:
                        numsubjobs = 0
                    helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
            except:
                pass


        #here we iterate over the lookup_input list and unprepare as necessary.
        for item in lookup_input:
            logger.info('Unpreparing objects referencing ShareDir %s' % item)
            self.lookup(sharedir=item, unprepare=True)

        #check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0 
        #so the user is made aware of them at shutdown
        for dir in os.listdir(ShareDir._root_shared_path):
            if not self.name.has_key(dir) and rmdir is False:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % dir)
                self.name[dir] = 0
            elif not self.name.has_key(dir) and rmdir is True:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % dir)
                shutil.rmtree(os.path.join(ShareDir._root_shared_path,dir))

        self._setDirty()
        self._releaseWriteAccess()
예제 #4
0
    def rebuild(self, unprepare=True, rmdir=False):
        """Rebuild the shareref table. 
        Clears the shareref table and then rebuilds it by iterating over all Ganga Objects 
        in the Job, Box and Task repositories. If an object has a ShareDir associated with it, 
        that ShareDir is added into the shareref table (the reference counter being incremented 
        accordingly). If called with the optional parameter 'unprepare=False', objects whose
        ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
        is unprepare=True, i.e. the job/application would be unprepared.
        After all Job/Box/Task objects have been checked, the inverse operation is performed, 
        i.e., for each directory in the ShareDir repository, a check is done to ensure there 
        is a matching entry in the shareref table. If not, and the optional parameter 
        'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem. 
        Otherwise, it will be added to the shareref table with a reference count of zero; 
        this results in the directory being deleted upon Ganga exit.
        """
        self._getWriteAccess()
        from Ganga.GPI import jobs, box, tasks
        # clear the shareref table
        self.name = {}
        lookup_input = []

        from Ganga.GPIDev.Lib.File import getSharedPath

        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing: 'job'})
        for thing in box.select():
            objectlist.append({thing: 'box'})
        for thing in tasks.select():
            objectlist.append({thing: 'task'})

        for item in objectlist:
            shortname = None
            try:
                shortname = item.keys()[0].is_prepared.name
            except AttributeError as err:
                logger.debug("Err: %s" % str(err))
                try:
                    shortname = item.keys()[0].application.is_prepared.name
                except AttributeError as err2:
                    logger.debug("Err2: %s" % str(err2))
                    try:
                        shortname = item.keys(
                        )[0].analysis.application.is_prepared.name
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % str(err3))
                        pass
            try:
                if shortname is not None and shortname is not True:
                    if os.path.basename(shortname) != shortname:
                        self.to_relative(item.keys()[0].is_prepared)
                    try:
                        numsubjobs = len(item.keys()[0].subjobs.ids())
                    except Exception as err:
                        logger.debug("Path Error: %s" % str(err))
                        Ganga.Utility.logging.log_unknown_exception()
                        numsubjobs = 0
                    self.helper(shortname,
                                unp=unprepare,
                                numsubjobs=numsubjobs)
            except Exception as err:
                logger.debug("-Error: %s" % str(err))
                Ganga.Utility.logging.log_unknown_exception()
                pass

        # here we iterate over the lookup_input list and unprepare as
        # necessary.
        for item in lookup_input:
            logger.info('Unpreparing objects referencing ShareDir %s' % item)
            self.lookup(sharedir=item, unprepare=True)

        # check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
        # so the user is made aware of them at shutdown
        for this_dir in os.listdir(getSharedPath()):
            if this_dir not in self.__getName().keys() and rmdir is False:
                logger.debug(
                    "%s isn't referenced by a GangaObject in the Job or Box repository."
                    % this_dir)
                self.__getName()[this_dir] = 0
            elif this_dir not in self.__getName().keys() and rmdir is True:
                logger.debug(
                    "%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory."
                    % this_dir)
                shutil.rmtree(os.path.join(getSharedPath(), this_dir))

        self._setDirty()
        self._releaseWriteAccess()
예제 #5
0
    def lookup(self, sharedir, unprepare=False):
        """
        Report Job, Box and Task repository items which reference a given ShareDir object. 
        The optional parameter 'unprepare=True' can be set to call the unprepare method 
        on the returned objects.
        """
        from Ganga.GPI import jobs, box, tasks
        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing: 'job'})
        for thing in box.select():
            objectlist.append({thing: 'box'})
        for thing in tasks.select():
            objectlist.append({thing: 'task'})

        run_unp = None
        master_index = 0
        for item in objectlist:
            try:
                item.keys()[0].is_prepared.name
                if item.keys()[0].is_prepared.name == sharedir:
                    logger.info(
                        'ShareDir %s is referenced by item #%s in %s repository'
                        % (sharedir, stripProxy(
                            item.keys()[0])._registry_id, item.values()[0]))
                    run_unp = item.keys()[0]
                    master_index += 1

            except AttributeError as err:
                logger.debug("Err: %s" % str(err))
                try:
                    item.keys()[0].application.is_prepared.name
                    if item.keys()[0].application.is_prepared.name == sharedir:
                        logger.info(
                            'ShareDir %s is referenced by item #%s in %s repository'
                            %
                            (sharedir, stripProxy(item.keys()[0])._registry_id,
                             item.values()[0]))
                        run_unp = item.keys()[0].application
                        master_index += 1
                except AttributeError as err2:
                    logger.debug("Err2: %s" % str(err2))
                    try:
                        item.keys()[0].analysis.application.is_prepared.name
                        if item.keys(
                        )[0].analysis.application.is_prepared.name == sharedir:
                            logger.info(
                                'ShareDir %s is referenced by item #%s in %s repository'
                                % (sharedir, stripProxy(
                                    item.keys()[0])._registry_id,
                                   item.values()[0]))
                            run_unp = item.keys()[0].analysis.application
                            master_index += 1
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % str(err3))
                        pass

            if run_unp is not None and unprepare is True:
                logger.info(
                    'Unpreparing %s repository object #%s associated with ShareDir %s'
                    % (item.values()[0], stripProxy(
                        item.keys()[0])._registry_id, sharedir))
                #                stripProxy(item.keys()[0]).unprepare()
                run_unp.unprepare()
                run_unp = None

        if unprepare is not True:
            logger.info('%s item(s) found referencing ShareDir %s',
                        master_index, sharedir)