Exemple #1
0
    def get_graph( self, group_nodes=[], ungroup_nodes=[],
            ungroup_recursive=False, ungroup_all=False, group_all=False ):
        family_nodes = self.suiterc.get_first_parent_descendants().keys()
        graphed_family_nodes = self.suiterc.triggering_families
        suite_polling_tasks = self.suiterc.suite_polling_tasks

        if self.ctime != None and self.stop_after != None:
            one = self.ctime
            two = self.stop_after
        else:
            one = str( self.suiterc.cfg['visualization']['initial cycle time'])
            two = str(self.suiterc.cfg['visualization']['final cycle time'])

        # TODO: move ct().get() out of this call (for error checking):
        graph = self.suiterc.get_graph( ct(one).get(), ct(two).get(),
                raw=self.raw, group_nodes=group_nodes,
                ungroup_nodes=ungroup_nodes,
                ungroup_recursive=ungroup_recursive,
                group_all=group_all, ungroup_all=ungroup_all,
                ignore_suicide=self.ignore_suicide )

        graph.graph_attr['rankdir'] = self.orientation

        for node in graph.nodes():
            name, tag = node.get_name().split(TaskID.DELIM)
            if name in family_nodes:
                if name in graphed_family_nodes:
                    node.attr['shape'] = 'doubleoctagon'
                else:
                    node.attr['shape'] = 'tripleoctagon'

        self.set_dotcode( graph.string() )
        self.graph = graph
Exemple #2
0
    def get_graph( self, group_nodes=[], ungroup_nodes=[], 
            ungroup_recursive=False, ungroup_all=False, group_all=False ):
        family_nodes = self.suiterc.members.keys()
        graphed_family_nodes = self.suiterc.families_used_in_graph

        if self.ctime != None and self.stop_after != None:
            one = self.ctime
            two = self.stop_after
        else:
            one = str( self.suiterc['visualization']['initial cycle time'])
            two = str(self.suiterc['visualization']['final cycle time'])

        # TO DO: move ct().get() out of this call (for error checking): 
        graph = self.suiterc.get_graph( ct(one).get(), ct(two).get(), 
                colored=True, raw=self.raw, 
                group_nodes=group_nodes, ungroup_nodes=ungroup_nodes, 
                ungroup_recursive=ungroup_recursive, 
                group_all=group_all, ungroup_all=ungroup_all )

        for node in graph.nodes():
            name, tag = node.get_name().split('%')
            if name in family_nodes:
                if name in graphed_family_nodes:
                    node.attr['shape'] = 'doubleoctagon'
                else:
                    node.attr['shape'] = 'doublecircle'

        self.set_dotcode( graph.string() )
        if self.outfile and not self.disable_output_image:
            try:
                graph.draw( self.outfile, prog='dot' )
            except IOError, x:
                print >> sys.stderr, x
                self.disable_output_image = True
Exemple #3
0
    def get_graph( self, group_nodes=[], ungroup_nodes=[], 
            ungroup_recursive=False, ungroup_all=False, group_all=False ):
        family_nodes = self.suiterc.get_first_parent_descendants().keys()
        graphed_family_nodes = self.suiterc.triggering_families
        suite_polling_tasks = self.suiterc.suite_polling_tasks

        if self.ctime != None and self.stop_after != None:
            one = self.ctime
            two = self.stop_after
        else:
            one = str( self.suiterc.cfg['visualization']['initial cycle time'])
            two = str(self.suiterc.cfg['visualization']['final cycle time'])

        # TODO: move ct().get() out of this call (for error checking): 
        graph = self.suiterc.get_graph( ct(one).get(), ct(two).get(),
                raw=self.raw, group_nodes=group_nodes,
                ungroup_nodes=ungroup_nodes, 
                ungroup_recursive=ungroup_recursive, 
                group_all=group_all, ungroup_all=ungroup_all,
                ignore_suicide=self.ignore_suicide )

        graph.graph_attr['rankdir'] = self.orientation

        for node in graph.nodes():
            name, tag = node.get_name().split(TaskID.DELIM)
            if name in family_nodes:
                if name in graphed_family_nodes:
                    node.attr['shape'] = 'doubleoctagon'
                else:
                    node.attr['shape'] = 'tripleoctagon'

        self.set_dotcode( graph.string() )
        self.graph = graph
Exemple #4
0
 def __init__( self, source, match, oper, ctime, offset, dest=None, 
         verbose=False, debug=False, mode=None, cheap=False ):
     self.source = source
     self.match = match
     self.ctime = ctime
     try:
         # check the validity of the base cycle time
         ct(ctime)
     except CycleTimeError,x:
         raise HousekeepingError, str(x)
Exemple #5
0
        def tclass_add_prerequisites(sself, startup, tag):
            # NOTE: Task objects hold all triggers defined for the task
            # in all cycling graph sections in this data structure:
            #     self.triggers[cycler] = [list of triggers for this cycler]
            # The list of triggers associated with cyclerX will only be
            # used by a particular task if the task's cycle time is a
            # valid member of cyclerX's sequence of cycle times.

            # 1) non-conditional triggers
            pp = plain_prerequisites(sself.id, self.ict)
            sp = plain_prerequisites(sself.id, self.ict)
            lp = loose_prerequisites(sself.id, self.ict)
            for cyc in self.triggers:
                for trig in self.triggers[cyc]:
                    if trig.startup and not startup:
                        continue
                    if trig.cycling and not cyc.valid(ct(sself.tag)):
                        # This trigger is not used in current cycle (see NOTE just above)
                        continue
                    # NOTE that if we need to check validity of async
                    # tags, async tasks can appear in cycling sections
                    # in which case cyc.valid( at(sself.tag)) will fail.
                    if trig.async_repeating:
                        lp.add(trig.get(tag, cyc))
                    else:
                        if trig.suicide:
                            sp.add(trig.get(tag, cyc))
                        else:
                            pp.add(trig.get(tag, cyc))
            sself.prerequisites.add_requisites(pp)
            sself.prerequisites.add_requisites(lp)
            sself.suicide_prerequisites.add_requisites(sp)

            # 2) conditional triggers
            for cyc in self.cond_triggers.keys():
                for ctrig, exp in self.cond_triggers[cyc]:
                    foo = ctrig.keys()[0]
                    if ctrig[foo].startup and not startup:
                        continue
                    if ctrig[foo].cycling and not cyc.valid(ct(sself.tag)):
                        # This trigger is not valid for current cycle (see NOTE just above)
                        continue
                    # NOTE that if we need to check validity of async
                    # tags, async tasks can appear in cycling sections
                    # in which case cyc.valid( at(sself.tag)) will fail.
                    cp = conditional_prerequisites(sself.id, self.ict)
                    for label in ctrig:
                        trig = ctrig[label]
                        cp.add(trig.get(tag, cyc), label)
                    cp.set_condition(exp)
                    if ctrig[foo].suicide:
                        sself.suicide_prerequisites.add_requisites(cp)
                    else:
                        sself.prerequisites.add_requisites(cp)
Exemple #6
0
        def tclass_add_prerequisites( sself, startup, tag  ):
            # NOTE: Task objects hold all triggers defined for the task
            # in all cycling graph sections in this data structure:
            #     self.triggers[cycler] = [list of triggers for this cycler]
            # The list of triggers associated with cyclerX will only be
            # used by a particular task if the task's cycle time is a
            # valid member of cyclerX's sequence of cycle times.

            # 1) non-conditional triggers
            pp = plain_prerequisites( sself.id, self.ict )
            sp = plain_prerequisites( sself.id, self.ict )
            lp = loose_prerequisites( sself.id, self.ict )
            for cyc in self.triggers:
                for trig in self.triggers[ cyc ]:
                    if trig.startup and not startup:
                            continue
                    if trig.cycling and not cyc.valid( ct(sself.tag) ):
                        # This trigger is not used in current cycle (see NOTE just above)
                        continue
                    # NOTE that if we need to check validity of async
                    # tags, async tasks can appear in cycling sections
                    # in which case cyc.valid( at(sself.tag)) will fail.
                    if trig.async_repeating:
                        lp.add( trig.get( tag, cyc ))
                    else:
                        if trig.suicide:
                            sp.add( trig.get( tag, cyc ))
                        else:
                            pp.add( trig.get( tag, cyc))
            sself.prerequisites.add_requisites( pp )
            sself.prerequisites.add_requisites( lp )
            sself.suicide_prerequisites.add_requisites( sp )

            # 2) conditional triggers
            for cyc in self.cond_triggers.keys():
                for ctrig, exp in self.cond_triggers[ cyc ]:
                    foo = ctrig.keys()[0]
                    if ctrig[foo].startup and not startup:
                        continue
                    if ctrig[foo].cycling and not cyc.valid( ct(sself.tag)):
                        # This trigger is not valid for current cycle (see NOTE just above)
                        continue
                    # NOTE that if we need to check validity of async
                    # tags, async tasks can appear in cycling sections
                    # in which case cyc.valid( at(sself.tag)) will fail.
                    cp = conditional_prerequisites( sself.id, self.ict )
                    for label in ctrig:
                        trig = ctrig[label]
                        cp.add( trig.get( tag, cyc ), label )
                    cp.set_condition( exp )
                    if ctrig[foo].suicide:
                        sself.suicide_prerequisites.add_requisites( cp )
                    else:
                        sself.prerequisites.add_requisites( cp )
Exemple #7
0
    def action( self, batchsize ):
        src_entries = 0
        matched = 0
        not_matched = 0
        total = 0
        actioned = 0
        print "________________________________________________________________________"
        print "SOURCE:", self.source
        if self.destn:
            print "TARGET:", self.destn
        print "MATCH :", self.match
        print "ACTION:", self.opern
        foo = ct( self.ctime )
        foo.decrement( hours=self.offset )
        print "CUTOFF:", self.ctime, '-', self.offset, '=', foo.get()
        batch = batchproc( batchsize, verbose=self.verbose )
        for entry in os.listdir( self.source ):
            src_entries += 1
            entrypath = os.path.join( self.source, entry )
            item = hkitem( entrypath, self.match, self.opern, self.ctime, self.offset, 
                    self.destn, self.mode, self.debug, self.cheap )
            if not item.matches():
                not_matched += 1
                continue
            matched += 1
            item.interpolate_destination()
            actioned += batch.add_or_process( item )
        actioned += batch.process()
 
        print 'MATCHED :', str(matched) + '/' + str(src_entries)
        print 'ACTIONED:', str(actioned) + '/' + str(matched)
Exemple #8
0
    def action(self, batchsize):
        src_entries = 0
        matched = 0
        not_matched = 0
        total = 0
        actioned = 0
        print "________________________________________________________________________"
        print "SOURCE:", self.source
        if self.destn:
            print "TARGET:", self.destn
        print "MATCH :", self.match
        print "ACTION:", self.opern
        foo = ct(self.ctime)
        foo.decrement(hours=self.offset)
        print "CUTOFF:", self.ctime, '-', self.offset, '=', foo.get()
        batch = batchproc(batchsize, verbose=self.verbose)
        for entry in os.listdir(self.source):
            src_entries += 1
            entrypath = os.path.join(self.source, entry)
            item = hkitem(entrypath, self.match, self.opern, self.ctime,
                          self.offset, self.destn, self.mode, self.debug,
                          self.cheap)
            if not item.matches():
                not_matched += 1
                continue
            matched += 1
            item.interpolate_destination()
            actioned += batch.add_or_process(item)
        actioned += batch.process()

        print 'MATCHED :', str(matched) + '/' + str(src_entries)
        print 'ACTIONED:', str(actioned) + '/' + str(matched)
Exemple #9
0
 def __init__(self,
              source,
              match,
              oper,
              ctime,
              offset,
              dest=None,
              verbose=False,
              debug=False,
              mode=None,
              cheap=False):
     self.source = source
     self.match = match
     self.ctime = ctime
     try:
         # check the validity of the base cycle time
         ct(ctime)
     except CycleTimeError, x:
         raise HousekeepingError, str(x)
Exemple #10
0
    def update_cycling( self, task, oldest ):
        # stop if all tasks are more than cutoff hours beyond suite start time
        if self.start_tag:
            st = ct( self.start_tag )
        else:
            st = ct( self.initial_oldest_ctime )

        ot = ct( oldest )
        delta1 = ot.subtract( st )
        delta2 = datetime.timedelta( 0, 0, 0, 0, 0, self.cutoff, 0 )
        if delta1 >= delta2:
            self.finalize()
            return
        # ignore task if its ctime more than configured hrs beyond suite start time?
        st = st
        tt = ct( task.c_time )
        delta1 = tt.subtract(st)
        if delta1 >= delta2:
            return
        for id in task.get_resolved_dependencies():
            l = id
            r = task.id
            self.graph.add_edge( l,r,False )
            self.write()
Exemple #11
0
    def update_cycling(self, task, oldest):
        # stop if all tasks are more than cutoff hours beyond suite start time
        if self.start_tag:
            st = ct(self.start_tag)
        else:
            st = ct(self.initial_oldest_ctime)

        ot = ct(oldest)
        delta1 = ot.subtract(st)
        delta2 = datetime.timedelta(0, 0, 0, 0, 0, self.cutoff, 0)
        if delta1 >= delta2:
            self.finalize()
            return
        # ignore task if its ctime more than configured hrs beyond suite start time?
        st = st
        tt = ct(task.c_time)
        delta1 = tt.subtract(st)
        if delta1 >= delta2:
            return
        for id in task.get_resolved_dependencies():
            l = id
            r = task.id
            self.graph.add_edge(l, r, False)
            self.write()
Exemple #12
0
 def get_stop_tag(self):
     found = False
     for line in self.lines:
         m = re.search('Stop tag: (.*)$', line)
         if m:
             found = True
             tag = m.groups()[0]
             if tag == "None":
                 return None
             else:
                 tag = ct(tag).get()
             break
     if found:
         return tag
     else:
         raise LogAnalyserError("ERROR: logged stop tag not found")
Exemple #13
0
 def get_stop_tag( self ):
     found = False
     for line in self.lines:
         m = re.search( 'Stop tag: (.*)$',line)
         if m:
             found = True
             tag = m.groups()[0]
             if tag == "None":
                 return None
             else:
                 tag = ct(tag).get()
             break
     if found:
         return tag
     else:
         raise LogAnalyserError( "ERROR: logged stop tag not found" )
Exemple #14
0
    def matches( self ):
        if self.debug:
            print "\nSource item:", self.path

        # does path match pattern
        m = re.search( self.pattern, self.path )
        if not m:
            if self.debug:
                print " + does not match"
            return False

        if self.debug:
            print " + MATCH"

        # extract cycle time from path
        mgrps = m.groups()
        if len(mgrps) == 1:
            self.matched_ctime = mgrps[0]
        elif len(mgrps) == 2:
            foo, bar = mgrps
            if len(foo) == 8 and len(bar) == 2:
                self.matched_ctime = foo + bar
            elif len(foo) == 2 and len(bar) == 8:
                self.matched_ctime = bar + foo
            else:
                print "WARNING: Housekeeping match problem:"
                print " + path: "+ self.path
                print " + pattern: " + self.pattern
                print " > extracted time groups:", m.groups()
                return False
        else:
            print "WARNING: Housekeeping match problem:"
            print " + path: "+ self.path
            print " + pattern: " + self.pattern
            print " > extracted time groups:", m.groups()
            return False

        # check validity of extracted cycle time
        try:
            ct(self.matched_ctime)
        except:
            if self.debug:
                print " + extracted cycle time is NOT VALID: " + self.matched_ctime
            return False
        else:
            if self.debug:
                print " + extracted cycle time: " + self.matched_ctime

        # assume ctime is >= self.matched_ctime
        foo = ct( self.ctime )
        bar = ct( self.matched_ctime )
        # gap hours
        gap = foo.subtract_hrs( bar )

        if self.debug:
            print " + computed offset hours", gap,
        if int(gap) < int(self.offset):
            if self.debug:
                print "- ignoring (does not make the cutoff)"
            return False
        
        if self.debug:
            print "- ACTIONABLE (does make the cutoff)"
        return True
Exemple #15
0
    def matches(self):
        if self.debug:
            print "\nSource item:", self.path

        # does path match pattern
        m = re.search(self.pattern, self.path)
        if not m:
            if self.debug:
                print " + does not match"
            return False

        if self.debug:
            print " + MATCH"

        # extract cycle time from path
        mgrps = m.groups()
        if len(mgrps) == 1:
            self.matched_ctime = mgrps[0]
        elif len(mgrps) == 2:
            foo, bar = mgrps
            if len(foo) == 8 and len(bar) == 2:
                self.matched_ctime = foo + bar
            elif len(foo) == 2 and len(bar) == 8:
                self.matched_ctime = bar + foo
            else:
                print "WARNING: Housekeeping match problem:"
                print " + path: " + self.path
                print " + pattern: " + self.pattern
                print " > extracted time groups:", m.groups()
                return False
        else:
            print "WARNING: Housekeeping match problem:"
            print " + path: " + self.path
            print " + pattern: " + self.pattern
            print " > extracted time groups:", m.groups()
            return False

        # check validity of extracted cycle time
        try:
            ct(self.matched_ctime)
        except:
            if self.debug:
                print " + extracted cycle time is NOT VALID: " + self.matched_ctime
            return False
        else:
            if self.debug:
                print " + extracted cycle time: " + self.matched_ctime

        # assume ctime is >= self.matched_ctime
        foo = ct(self.ctime)
        bar = ct(self.matched_ctime)
        # gap hours
        gap = foo.subtract_hrs(bar)

        if self.debug:
            print " + computed offset hours", gap,
        if int(gap) < int(self.offset):
            if self.debug:
                print "- ignoring (does not make the cutoff)"
            return False

        if self.debug:
            print "- ACTIONABLE (does make the cutoff)"
        return True
Exemple #16
0
        def tclass_add_prerequisites( sself, startup, tag  ):
            # NOTE: Task objects hold all triggers defined for the task
            # in all cycling graph sections in this data structure:
            #     self.triggers[cycler] = [list of triggers for this cycler]
            # The list of triggers associated with cyclerX will only be
            # used by a particular task if the task's cycle time is a
            # valid member of cyclerX's sequence of cycle times.

            # 1) non-conditional triggers
            pp = plain_prerequisites( sself.id, self.ict )
            sp = plain_prerequisites( sself.id, self.ict )
            lp = loose_prerequisites( sself.id, self.ict )

            if self.sequential:
                # For tasks declared 'sequential' we automatically add a
                # previous-instance inter-cycle trigger, and adjust the
                # cleanup cutoff (determined by inter-cycle triggers)
                # accordingly.
                pp.add( sself.name + '.' + sself.cycon.prev( sself.c_time ) + ' succeeded' )
                next_inst_ct = sself.cycon.next( sself.c_time )
                if int(sself.cleanup_cutoff) < int(next_inst_ct):
                    sself.cleanup_cutoff = next_inst_ct

            for cyc in self.triggers:
                for trig in self.triggers[ cyc ]:
                    if trig.startup and not startup:
                            continue
                    if trig.cycling and not cyc.valid( ct(sself.tag) ):
                        # This trigger is not used in current cycle (see NOTE just above)
                        continue
                    # NOTE that if we need to check validity of async
                    # tags, async tasks can appear in cycling sections
                    # in which case cyc.valid( at(sself.tag)) will fail.
                    if trig.async_repeating:
                        lp.add( trig.get( tag, cyc ))
                    elif self.ict is None or \
                            trig.evaluation_offset is None or \
                                (int(tag) - int(trig.evaluation_offset)) >= int(self.ict):
                            # i.c.t. can be None after a restart, if one
                            # is not specified in the suite definition.
                            if trig.suicide:
                                sp.add( trig.get( tag, cyc ))
                            else:
                                pp.add( trig.get( tag, cyc))
            sself.prerequisites.add_requisites( pp )
            sself.prerequisites.add_requisites( lp )
            sself.suicide_prerequisites.add_requisites( sp )

            # 2) conditional triggers
            for cyc in self.cond_triggers.keys():
                for ctrig, exp in self.cond_triggers[ cyc ]:
                    foo = ctrig.keys()[0]
                    if ctrig[foo].startup and not startup:
                        continue
                    if ctrig[foo].cycling and not cyc.valid( ct(sself.tag)):
                        # This trigger is not valid for current cycle (see NOTE just above)
                        continue
                    # NOTE that if we need to check validity of async
                    # tags, async tasks can appear in cycling sections
                    # in which case cyc.valid( at(sself.tag)) will fail.
                    cp = conditional_prerequisites( sself.id, self.ict )
                    for label in ctrig:
                        trig = ctrig[label]
                        if self.ict is not None and trig.evaluation_offset is not None:
                            cp.add( trig.get( tag, cyc ), label,
                                    (int(tag) - int(trig.evaluation_offset)) < int(self.ict))
                        else:
                            cp.add( trig.get( tag, cyc ), label )
                    cp.set_condition( exp )
                    if ctrig[foo].suicide:
                        sself.suicide_prerequisites.add_requisites( cp )
                    else:
                        sself.prerequisites.add_requisites( cp )