def get_graph( self, group_nodes=[], ungroup_nodes=[],
ungroup_recursive=False, ungroup_all=False, group_all=False ):
family_nodes = self.suiterc.get_first_parent_descendants().keys()
graphed_family_nodes = self.suiterc.triggering_families
suite_polling_tasks = self.suiterc.suite_polling_tasks
if self.ctime != None and self.stop_after != None:
one = self.ctime
two = self.stop_after
else:
one = str( self.suiterc.cfg['visualization']['initial cycle time'])
two = str(self.suiterc.cfg['visualization']['final cycle time'])
# TODO: move ct().get() out of this call (for error checking):
graph = self.suiterc.get_graph( ct(one).get(), ct(two).get(),
raw=self.raw, group_nodes=group_nodes,
ungroup_nodes=ungroup_nodes,
ungroup_recursive=ungroup_recursive,
group_all=group_all, ungroup_all=ungroup_all,
ignore_suicide=self.ignore_suicide )
graph.graph_attr['rankdir'] = self.orientation
for node in graph.nodes():
name, tag = node.get_name().split(TaskID.DELIM)
if name in family_nodes:
if name in graphed_family_nodes:
node.attr['shape'] = 'doubleoctagon'
else:
node.attr['shape'] = 'tripleoctagon'
self.set_dotcode( graph.string() )
self.graph = graph
def get_graph( self, group_nodes=[], ungroup_nodes=[],
ungroup_recursive=False, ungroup_all=False, group_all=False ):
family_nodes = self.suiterc.members.keys()
graphed_family_nodes = self.suiterc.families_used_in_graph
if self.ctime != None and self.stop_after != None:
one = self.ctime
two = self.stop_after
else:
one = str( self.suiterc['visualization']['initial cycle time'])
two = str(self.suiterc['visualization']['final cycle time'])
# TO DO: move ct().get() out of this call (for error checking):
graph = self.suiterc.get_graph( ct(one).get(), ct(two).get(),
colored=True, raw=self.raw,
group_nodes=group_nodes, ungroup_nodes=ungroup_nodes,
ungroup_recursive=ungroup_recursive,
group_all=group_all, ungroup_all=ungroup_all )
for node in graph.nodes():
name, tag = node.get_name().split('%')
if name in family_nodes:
if name in graphed_family_nodes:
node.attr['shape'] = 'doubleoctagon'
else:
node.attr['shape'] = 'doublecircle'
self.set_dotcode( graph.string() )
if self.outfile and not self.disable_output_image:
try:
graph.draw( self.outfile, prog='dot' )
except IOError, x:
print >> sys.stderr, x
self.disable_output_image = True
def get_graph( self, group_nodes=[], ungroup_nodes=[],
ungroup_recursive=False, ungroup_all=False, group_all=False ):
family_nodes = self.suiterc.get_first_parent_descendants().keys()
graphed_family_nodes = self.suiterc.triggering_families
suite_polling_tasks = self.suiterc.suite_polling_tasks
if self.ctime != None and self.stop_after != None:
one = self.ctime
two = self.stop_after
else:
one = str( self.suiterc.cfg['visualization']['initial cycle time'])
two = str(self.suiterc.cfg['visualization']['final cycle time'])
# TODO: move ct().get() out of this call (for error checking):
graph = self.suiterc.get_graph( ct(one).get(), ct(two).get(),
raw=self.raw, group_nodes=group_nodes,
ungroup_nodes=ungroup_nodes,
ungroup_recursive=ungroup_recursive,
group_all=group_all, ungroup_all=ungroup_all,
ignore_suicide=self.ignore_suicide )
graph.graph_attr['rankdir'] = self.orientation
for node in graph.nodes():
name, tag = node.get_name().split(TaskID.DELIM)
if name in family_nodes:
if name in graphed_family_nodes:
node.attr['shape'] = 'doubleoctagon'
else:
node.attr['shape'] = 'tripleoctagon'
self.set_dotcode( graph.string() )
self.graph = graph
def __init__( self, source, match, oper, ctime, offset, dest=None,
verbose=False, debug=False, mode=None, cheap=False ):
self.source = source
self.match = match
self.ctime = ctime
try:
# check the validity of the base cycle time
ct(ctime)
except CycleTimeError,x:
raise HousekeepingError, str(x)
def tclass_add_prerequisites(sself, startup, tag):
# NOTE: Task objects hold all triggers defined for the task
# in all cycling graph sections in this data structure:
# self.triggers[cycler] = [list of triggers for this cycler]
# The list of triggers associated with cyclerX will only be
# used by a particular task if the task's cycle time is a
# valid member of cyclerX's sequence of cycle times.
# 1) non-conditional triggers
pp = plain_prerequisites(sself.id, self.ict)
sp = plain_prerequisites(sself.id, self.ict)
lp = loose_prerequisites(sself.id, self.ict)
for cyc in self.triggers:
for trig in self.triggers[cyc]:
if trig.startup and not startup:
continue
if trig.cycling and not cyc.valid(ct(sself.tag)):
# This trigger is not used in current cycle (see NOTE just above)
continue
# NOTE that if we need to check validity of async
# tags, async tasks can appear in cycling sections
# in which case cyc.valid( at(sself.tag)) will fail.
if trig.async_repeating:
lp.add(trig.get(tag, cyc))
else:
if trig.suicide:
sp.add(trig.get(tag, cyc))
else:
pp.add(trig.get(tag, cyc))
sself.prerequisites.add_requisites(pp)
sself.prerequisites.add_requisites(lp)
sself.suicide_prerequisites.add_requisites(sp)
# 2) conditional triggers
for cyc in self.cond_triggers.keys():
for ctrig, exp in self.cond_triggers[cyc]:
foo = ctrig.keys()[0]
if ctrig[foo].startup and not startup:
continue
if ctrig[foo].cycling and not cyc.valid(ct(sself.tag)):
# This trigger is not valid for current cycle (see NOTE just above)
continue
# NOTE that if we need to check validity of async
# tags, async tasks can appear in cycling sections
# in which case cyc.valid( at(sself.tag)) will fail.
cp = conditional_prerequisites(sself.id, self.ict)
for label in ctrig:
trig = ctrig[label]
cp.add(trig.get(tag, cyc), label)
cp.set_condition(exp)
if ctrig[foo].suicide:
sself.suicide_prerequisites.add_requisites(cp)
else:
sself.prerequisites.add_requisites(cp)
def tclass_add_prerequisites( sself, startup, tag ):
# NOTE: Task objects hold all triggers defined for the task
# in all cycling graph sections in this data structure:
# self.triggers[cycler] = [list of triggers for this cycler]
# The list of triggers associated with cyclerX will only be
# used by a particular task if the task's cycle time is a
# valid member of cyclerX's sequence of cycle times.
# 1) non-conditional triggers
pp = plain_prerequisites( sself.id, self.ict )
sp = plain_prerequisites( sself.id, self.ict )
lp = loose_prerequisites( sself.id, self.ict )
for cyc in self.triggers:
for trig in self.triggers[ cyc ]:
if trig.startup and not startup:
continue
if trig.cycling and not cyc.valid( ct(sself.tag) ):
# This trigger is not used in current cycle (see NOTE just above)
continue
# NOTE that if we need to check validity of async
# tags, async tasks can appear in cycling sections
# in which case cyc.valid( at(sself.tag)) will fail.
if trig.async_repeating:
lp.add( trig.get( tag, cyc ))
else:
if trig.suicide:
sp.add( trig.get( tag, cyc ))
else:
pp.add( trig.get( tag, cyc))
sself.prerequisites.add_requisites( pp )
sself.prerequisites.add_requisites( lp )
sself.suicide_prerequisites.add_requisites( sp )
# 2) conditional triggers
for cyc in self.cond_triggers.keys():
for ctrig, exp in self.cond_triggers[ cyc ]:
foo = ctrig.keys()[0]
if ctrig[foo].startup and not startup:
continue
if ctrig[foo].cycling and not cyc.valid( ct(sself.tag)):
# This trigger is not valid for current cycle (see NOTE just above)
continue
# NOTE that if we need to check validity of async
# tags, async tasks can appear in cycling sections
# in which case cyc.valid( at(sself.tag)) will fail.
cp = conditional_prerequisites( sself.id, self.ict )
for label in ctrig:
trig = ctrig[label]
cp.add( trig.get( tag, cyc ), label )
cp.set_condition( exp )
if ctrig[foo].suicide:
sself.suicide_prerequisites.add_requisites( cp )
else:
sself.prerequisites.add_requisites( cp )
def action( self, batchsize ):
src_entries = 0
matched = 0
not_matched = 0
total = 0
actioned = 0
print "________________________________________________________________________"
print "SOURCE:", self.source
if self.destn:
print "TARGET:", self.destn
print "MATCH :", self.match
print "ACTION:", self.opern
foo = ct( self.ctime )
foo.decrement( hours=self.offset )
print "CUTOFF:", self.ctime, '-', self.offset, '=', foo.get()
batch = batchproc( batchsize, verbose=self.verbose )
for entry in os.listdir( self.source ):
src_entries += 1
entrypath = os.path.join( self.source, entry )
item = hkitem( entrypath, self.match, self.opern, self.ctime, self.offset,
self.destn, self.mode, self.debug, self.cheap )
if not item.matches():
not_matched += 1
continue
matched += 1
item.interpolate_destination()
actioned += batch.add_or_process( item )
actioned += batch.process()
print 'MATCHED :', str(matched) + '/' + str(src_entries)
print 'ACTIONED:', str(actioned) + '/' + str(matched)
def action(self, batchsize):
src_entries = 0
matched = 0
not_matched = 0
total = 0
actioned = 0
print "________________________________________________________________________"
print "SOURCE:", self.source
if self.destn:
print "TARGET:", self.destn
print "MATCH :", self.match
print "ACTION:", self.opern
foo = ct(self.ctime)
foo.decrement(hours=self.offset)
print "CUTOFF:", self.ctime, '-', self.offset, '=', foo.get()
batch = batchproc(batchsize, verbose=self.verbose)
for entry in os.listdir(self.source):
src_entries += 1
entrypath = os.path.join(self.source, entry)
item = hkitem(entrypath, self.match, self.opern, self.ctime,
self.offset, self.destn, self.mode, self.debug,
self.cheap)
if not item.matches():
not_matched += 1
continue
matched += 1
item.interpolate_destination()
actioned += batch.add_or_process(item)
actioned += batch.process()
print 'MATCHED :', str(matched) + '/' + str(src_entries)
print 'ACTIONED:', str(actioned) + '/' + str(matched)
def __init__(self,
source,
match,
oper,
ctime,
offset,
dest=None,
verbose=False,
debug=False,
mode=None,
cheap=False):
self.source = source
self.match = match
self.ctime = ctime
try:
# check the validity of the base cycle time
ct(ctime)
except CycleTimeError, x:
raise HousekeepingError, str(x)
def update_cycling( self, task, oldest ):
# stop if all tasks are more than cutoff hours beyond suite start time
if self.start_tag:
st = ct( self.start_tag )
else:
st = ct( self.initial_oldest_ctime )
ot = ct( oldest )
delta1 = ot.subtract( st )
delta2 = datetime.timedelta( 0, 0, 0, 0, 0, self.cutoff, 0 )
if delta1 >= delta2:
self.finalize()
return
# ignore task if its ctime more than configured hrs beyond suite start time?
st = st
tt = ct( task.c_time )
delta1 = tt.subtract(st)
if delta1 >= delta2:
return
for id in task.get_resolved_dependencies():
l = id
r = task.id
self.graph.add_edge( l,r,False )
self.write()
def update_cycling(self, task, oldest):
# stop if all tasks are more than cutoff hours beyond suite start time
if self.start_tag:
st = ct(self.start_tag)
else:
st = ct(self.initial_oldest_ctime)
ot = ct(oldest)
delta1 = ot.subtract(st)
delta2 = datetime.timedelta(0, 0, 0, 0, 0, self.cutoff, 0)
if delta1 >= delta2:
self.finalize()
return
# ignore task if its ctime more than configured hrs beyond suite start time?
st = st
tt = ct(task.c_time)
delta1 = tt.subtract(st)
if delta1 >= delta2:
return
for id in task.get_resolved_dependencies():
l = id
r = task.id
self.graph.add_edge(l, r, False)
self.write()
def get_stop_tag(self):
found = False
for line in self.lines:
m = re.search('Stop tag: (.*)$', line)
if m:
found = True
tag = m.groups()[0]
if tag == "None":
return None
else:
tag = ct(tag).get()
break
if found:
return tag
else:
raise LogAnalyserError("ERROR: logged stop tag not found")
def get_stop_tag( self ):
found = False
for line in self.lines:
m = re.search( 'Stop tag: (.*)$',line)
if m:
found = True
tag = m.groups()[0]
if tag == "None":
return None
else:
tag = ct(tag).get()
break
if found:
return tag
else:
raise LogAnalyserError( "ERROR: logged stop tag not found" )
def matches( self ):
if self.debug:
print "\nSource item:", self.path
# does path match pattern
m = re.search( self.pattern, self.path )
if not m:
if self.debug:
print " + does not match"
return False
if self.debug:
print " + MATCH"
# extract cycle time from path
mgrps = m.groups()
if len(mgrps) == 1:
self.matched_ctime = mgrps[0]
elif len(mgrps) == 2:
foo, bar = mgrps
if len(foo) == 8 and len(bar) == 2:
self.matched_ctime = foo + bar
elif len(foo) == 2 and len(bar) == 8:
self.matched_ctime = bar + foo
else:
print "WARNING: Housekeeping match problem:"
print " + path: "+ self.path
print " + pattern: " + self.pattern
print " > extracted time groups:", m.groups()
return False
else:
print "WARNING: Housekeeping match problem:"
print " + path: "+ self.path
print " + pattern: " + self.pattern
print " > extracted time groups:", m.groups()
return False
# check validity of extracted cycle time
try:
ct(self.matched_ctime)
except:
if self.debug:
print " + extracted cycle time is NOT VALID: " + self.matched_ctime
return False
else:
if self.debug:
print " + extracted cycle time: " + self.matched_ctime
# assume ctime is >= self.matched_ctime
foo = ct( self.ctime )
bar = ct( self.matched_ctime )
# gap hours
gap = foo.subtract_hrs( bar )
if self.debug:
print " + computed offset hours", gap,
if int(gap) < int(self.offset):
if self.debug:
print "- ignoring (does not make the cutoff)"
return False
if self.debug:
print "- ACTIONABLE (does make the cutoff)"
return True
def matches(self):
if self.debug:
print "\nSource item:", self.path
# does path match pattern
m = re.search(self.pattern, self.path)
if not m:
if self.debug:
print " + does not match"
return False
if self.debug:
print " + MATCH"
# extract cycle time from path
mgrps = m.groups()
if len(mgrps) == 1:
self.matched_ctime = mgrps[0]
elif len(mgrps) == 2:
foo, bar = mgrps
if len(foo) == 8 and len(bar) == 2:
self.matched_ctime = foo + bar
elif len(foo) == 2 and len(bar) == 8:
self.matched_ctime = bar + foo
else:
print "WARNING: Housekeeping match problem:"
print " + path: " + self.path
print " + pattern: " + self.pattern
print " > extracted time groups:", m.groups()
return False
else:
print "WARNING: Housekeeping match problem:"
print " + path: " + self.path
print " + pattern: " + self.pattern
print " > extracted time groups:", m.groups()
return False
# check validity of extracted cycle time
try:
ct(self.matched_ctime)
except:
if self.debug:
print " + extracted cycle time is NOT VALID: " + self.matched_ctime
return False
else:
if self.debug:
print " + extracted cycle time: " + self.matched_ctime
# assume ctime is >= self.matched_ctime
foo = ct(self.ctime)
bar = ct(self.matched_ctime)
# gap hours
gap = foo.subtract_hrs(bar)
if self.debug:
print " + computed offset hours", gap,
if int(gap) < int(self.offset):
if self.debug:
print "- ignoring (does not make the cutoff)"
return False
if self.debug:
print "- ACTIONABLE (does make the cutoff)"
return True
def tclass_add_prerequisites( sself, startup, tag ):
# NOTE: Task objects hold all triggers defined for the task
# in all cycling graph sections in this data structure:
# self.triggers[cycler] = [list of triggers for this cycler]
# The list of triggers associated with cyclerX will only be
# used by a particular task if the task's cycle time is a
# valid member of cyclerX's sequence of cycle times.
# 1) non-conditional triggers
pp = plain_prerequisites( sself.id, self.ict )
sp = plain_prerequisites( sself.id, self.ict )
lp = loose_prerequisites( sself.id, self.ict )
if self.sequential:
# For tasks declared 'sequential' we automatically add a
# previous-instance inter-cycle trigger, and adjust the
# cleanup cutoff (determined by inter-cycle triggers)
# accordingly.
pp.add( sself.name + '.' + sself.cycon.prev( sself.c_time ) + ' succeeded' )
next_inst_ct = sself.cycon.next( sself.c_time )
if int(sself.cleanup_cutoff) < int(next_inst_ct):
sself.cleanup_cutoff = next_inst_ct
for cyc in self.triggers:
for trig in self.triggers[ cyc ]:
if trig.startup and not startup:
continue
if trig.cycling and not cyc.valid( ct(sself.tag) ):
# This trigger is not used in current cycle (see NOTE just above)
continue
# NOTE that if we need to check validity of async
# tags, async tasks can appear in cycling sections
# in which case cyc.valid( at(sself.tag)) will fail.
if trig.async_repeating:
lp.add( trig.get( tag, cyc ))
elif self.ict is None or \
trig.evaluation_offset is None or \
(int(tag) - int(trig.evaluation_offset)) >= int(self.ict):
# i.c.t. can be None after a restart, if one
# is not specified in the suite definition.
if trig.suicide:
sp.add( trig.get( tag, cyc ))
else:
pp.add( trig.get( tag, cyc))
sself.prerequisites.add_requisites( pp )
sself.prerequisites.add_requisites( lp )
sself.suicide_prerequisites.add_requisites( sp )
# 2) conditional triggers
for cyc in self.cond_triggers.keys():
for ctrig, exp in self.cond_triggers[ cyc ]:
foo = ctrig.keys()[0]
if ctrig[foo].startup and not startup:
continue
if ctrig[foo].cycling and not cyc.valid( ct(sself.tag)):
# This trigger is not valid for current cycle (see NOTE just above)
continue
# NOTE that if we need to check validity of async
# tags, async tasks can appear in cycling sections
# in which case cyc.valid( at(sself.tag)) will fail.
cp = conditional_prerequisites( sself.id, self.ict )
for label in ctrig:
trig = ctrig[label]
if self.ict is not None and trig.evaluation_offset is not None:
cp.add( trig.get( tag, cyc ), label,
(int(tag) - int(trig.evaluation_offset)) < int(self.ict))
else:
cp.add( trig.get( tag, cyc ), label )
cp.set_condition( exp )
if ctrig[foo].suicide:
sself.suicide_prerequisites.add_requisites( cp )
else:
sself.prerequisites.add_requisites( cp )