def run_grid(mode, hosts_list, keytype, tasks, timeslot, brokenfile,
logfile, user=None, passwd=None, time_limit=86400):
if 0:
print 'run_grid ...'
print 'mode:', mode
print 'hosts_list:', hosts_list
print 'keytype:', keytype
print 'tasks:', tasks
print 'timeslot:', timeslot
print 'brokenfile:', brokenfile
print 'logfile:', logfile
print 'user:'******'passwd:', passwd
print 'time_limit:', time_limit
print ''
print ''
hm = HostsManager(hosts_list)
if len(hm._available_list) == 0:
msg.error("All hosts down :(")
return
if tasks is None : return
#print 'run_grid ...'
#print 'tasks:', tasks
brokenfd = open(brokenfile, 'w')
log = open(logfile, 'w')
user = User(user, passwd, keytype)
args = (timeslot, user, tasks, hm, log, brokenfd, time_limit)
if mode == 'dispatch': tm = DispatchedTasksManager(*args)
elif mode == 'repeat': tm = RepeatedTasksManager(*args)
elif mode in ['hie', 'hierarchic']:
tm = HierarchicalTasksManager(*args)
tm.start()
brokenfd.close()
log.close()
def read_hosts(hosts):
if hosts.find(',') == -1:
try:
fd = open(hosts, 'r')
except exceptions.IOError:
msg.error("host file '%s' can not be opened" % hosts)
else:
hosts = ' '.join(fd.readlines()).replace('\n', ' ')
fd.close()
hosts_list = re.split('[\s,]+', hosts)
if hosts_list[-1] == '': del hosts_list[-1]
return hosts_list
def read_hosts(hosts):
if hosts.find(',') == -1:
try:
fd = open(hosts, 'r')
except exceptions.IOError:
msg.error("host file '%s' can not be opened" % hosts)
else:
hosts = ' '.join(fd.readlines()).replace('\n', ' ')
fd.close()
hosts_list = re.split('[\s,]+', hosts)
if hosts_list[-1] == '': del hosts_list[-1]
return hosts_list
def parse_options(parser):
morehelpmap = {
'hosts': hosts_help,
'tasks': tasks_help,
'mode': mode_help,
'broken': broken_help,
'log': log_help,
'timeslot': timeslot_help
}
(options, args) = parser.parse_args(sys.argv)
error = False
error_msg = []
if options.morehelp != None:
try:
morehelpmap[options.morehelp](sys.argv[0])
sys.exit(1)
except KeyError:
msg.error("No help for '%s'.\n" % options.morehelp)
msg.write_list(
[' ', ('*', 'green'), ' List of available full helps :\n '])
for k in morehelpmap.keys():
msg.write('%s, ' % k)
msg.write('\n')
sys.exit(1)
mandatory_options = {'hosts': options.hosts}
timeslot = read_timeslot(options.timeslot)
if None in mandatory_options.values():
opt = [n for n, o in mandatory_options.items() if o == None]
error_msg.append('missing options : %s' % opt)
error = True
if not options.mode in ["dispatch", "repeat", "hie", "hierarchical"]:
error_msg.append("'%s' : unknown task manager mode." % \
options.mode)
error = True
if error:
parser.print_help()
for m in error_msg:
msg.error(m)
sys.exit(1)
hosts_list = read_hosts(options.hosts)
if options.tasks:
tasks_list = read_tasks(options.tasks, options.mode)
else:
tasks_list = None
return options, timeslot, hosts_list, tasks_list
def create_hietask(file):
tasks_dic = {}
read_file_hier(file, tasks_dic)
if len(tasks_dic) == 0:
msg.error("task file is empty")
sys.exit(1)
for v, (deps, tasks) in tasks_dic.items():
for d in deps:
if tasks_dic.has_key(d): continue
try:
hietask = create_hietask(d)
tasks_dic[d] = hietask
except exceptions.IOError:
e = "in file '" + file + \
"', rule or file '" + d + \
"' does not exist"
msg.error(e)
sys.exit(1)
return TaskHierarchical(file, tasks_dic)
def create_hietask(file):
tasks_dic = {}
read_file_hier(file, tasks_dic)
if len(tasks_dic) == 0:
msg.error("task file is empty")
sys.exit(1)
for v, (deps, tasks) in tasks_dic.items():
for d in deps:
if tasks_dic.has_key(d): continue
try:
hietask = create_hietask(d)
tasks_dic[d] = hietask
except exceptions.IOError:
e = "in file '" + file + \
"', rule or file '" + d + \
"' does not exist"
msg.error(e)
sys.exit(1)
return TaskHierarchical(file, tasks_dic)
def run_grid(mode,
hosts_list,
keytype,
tasks,
timeslot,
brokenfile=None,
logfile=None,
user=None,
passwd=None,
time_limit=86400):
if 0:
print 'run_grid ...'
print 'mode:', mode
print 'hosts_list:', hosts_list
print 'keytype:', keytype
print 'tasks:', tasks
print 'timeslot:', timeslot
print 'brokenfile:', brokenfile
print 'logfile:', logfile
print 'user:'******'passwd:', passwd
print 'time_limit:', time_limit
print ''
print ''
hm = HostsManager(hosts_list)
if len(hm._available_list) == 0:
msg.error("All hosts down :(")
return
if tasks is None: return
#print 'run_grid ...'
#print 'tasks:', tasks
brokenfd = open(brokenfile or os.devnull, 'w')
log = open(logfile or os.devnull, 'w')
user = User(user, passwd, keytype)
args = (timeslot, user, tasks, hm, log, brokenfd, time_limit)
if mode == 'dispatch': tm = DispatchedTasksManager(*args)
elif mode == 'repeat': tm = RepeatedTasksManager(*args)
elif mode in ['hie', 'hierarchic']:
tm = HierarchicalTasksManager(*args)
tm.start()
brokenfd.close()
log.close()
def parse_options(parser):
morehelpmap = {'hosts' : hosts_help, 'tasks' : tasks_help,
'mode' : mode_help, 'broken' : broken_help, 'log' : log_help,
'timeslot' : timeslot_help}
(options, args) = parser.parse_args(sys.argv)
error = False
error_msg = []
if options.morehelp != None:
try:
morehelpmap[options.morehelp](sys.argv[0])
sys.exit(1)
except KeyError:
msg.error("No help for '%s'.\n" % options.morehelp)
msg.write_list([' ', ('*', 'green'),
' List of available full helps :\n '])
for k in morehelpmap.keys(): msg.write('%s, ' % k)
msg.write('\n')
sys.exit(1)
mandatory_options = {'hosts' : options.hosts}
timeslot = read_timeslot(options.timeslot)
if None in mandatory_options.values():
opt = [n for n, o in mandatory_options.items() if o == None]
error_msg.append('missing options : %s' % opt)
error = True
if not options.mode in ["dispatch", "repeat", "hie", "hierarchical"]:
error_msg.append("'%s' : unknown task manager mode." % \
options.mode)
error = True
if error:
parser.print_help()
for m in error_msg: msg.error(m)
sys.exit(1)
hosts_list = read_hosts(options.hosts)
if options.tasks:
tasks_list = read_tasks(options.tasks, options.mode)
else: tasks_list = None
return options, timeslot, hosts_list, tasks_list
def read_timeslot(timeslot):
if timeslot in ( '-', 'allday' ):
return TimeSlot(0, 86400) # [0:24h]
elif timeslot == 'morning':
return TimeSlot(0, 43200) # [0:12h]
elif timeslot == 'afternoon':
return TimeSlot(43200, 86400) # [12h:24h]
elif timeslot == 'midi':
return TimeSlot(43200, 50400) # [12h:14h]
elif timeslot == 'night':
# [0:8h] + [20h:24h] = [20h:8h]
return TimeSlotList([TimeSlot(0, 28800),
TimeSlot(72000, 86400)])
timeslot_list = timeslot.split(',')
tsl = []
for one_timeslot in timeslot_list:
try:
t1, t2 = one_timeslot.split('-')
except ValueError:
msg.error("'%s' : bad timeslot format" % one_timeslot)
sys.exit(1)
seconds = []
for t in [t1, t2]:
try:
h, m, s = t.split(':')
except ValueError:
msg.error("'%s' : bad hour format" % t)
sys.exit(1)
h, m, s = int(h), int(m), int(s)
seconds.append(s + 60 * (m + 60 * h))
tsl.append(TimeSlot(seconds[0], seconds[1]))
if len(timeslot_list) > 1:
ts = TimeSlotList(tsl)
else: ts = tsl[0]
return ts
def read_timeslot(timeslot):
if timeslot in ('-', 'allday'):
return TimeSlot(0, 86400) # [0:24h]
elif timeslot == 'morning':
return TimeSlot(0, 43200) # [0:12h]
elif timeslot == 'afternoon':
return TimeSlot(43200, 86400) # [12h:24h]
elif timeslot == 'midi':
return TimeSlot(43200, 50400) # [12h:14h]
elif timeslot == 'night':
# [0:8h] + [20h:24h] = [20h:8h]
return TimeSlotList([TimeSlot(0, 28800), TimeSlot(72000, 86400)])
timeslot_list = timeslot.split(',')
tsl = []
for one_timeslot in timeslot_list:
try:
t1, t2 = one_timeslot.split('-')
except ValueError:
msg.error("'%s' : bad timeslot format" % one_timeslot)
sys.exit(1)
seconds = []
for t in [t1, t2]:
try:
h, m, s = t.split(':')
except ValueError:
msg.error("'%s' : bad hour format" % t)
sys.exit(1)
h, m, s = int(h), int(m), int(s)
seconds.append(s + 60 * (m + 60 * h))
tsl.append(TimeSlot(seconds[0], seconds[1]))
if len(timeslot_list) > 1:
ts = TimeSlotList(tsl)
else:
ts = tsl[0]
return ts
def read_hierarchic_tasks(tasks_file):
def read_file_list(file):
fd = open(file, 'r')
lines = fd.readlines()
tasks = TaskList(lines)
fd.close()
return tasks
def read_deps_tokens(line):
# task : dep1 dep2 dep3
taskre = re.compile('[ \t]*:[ \t]*')
try:
task, deps = re.split(taskre, line)
except ValueError:
return None
deps = deps.rstrip('\n')
depsre = re.compile('[\t ]+')
deps = re.split(depsre, deps)
if deps == ['']: deps = []
return task, deps
def read_file_hier(file, tasks_dic):
DEPS_STATE, TASKS_STATE = 0, 1
fd = open(file, 'r')
lines = fd.readlines()
state = DEPS_STATE
tasks = []
task = None
deps = None
for l in lines:
blankre = re.compile('^[ \t]*$')
if blankre.match(l): continue
depsre = re.compile('[a-zA-Z0-9_]*[ \t]*:[ \t]*')
if depsre.match(l):
state = DEPS_STATE
if state == DEPS_STATE:
if task: tasks_dic[task] = deps, tasks
tokens = read_deps_tokens(l)
state = TASKS_STATE
tasks = []
if tokens is None:
task, deps = "all", []
tasks.append(l)
else: task, deps = tokens
elif state == TASKS_STATE:
tasks.append(l)
if task is not None: tasks_dic[task] = deps, tasks
fd.close()
def create_hietask(file):
tasks_dic = {}
read_file_hier(file, tasks_dic)
if len(tasks_dic) == 0:
msg.error("task file is empty")
sys.exit(1)
for v, (deps, tasks) in tasks_dic.items():
for d in deps:
if tasks_dic.has_key(d): continue
try:
hietask = create_hietask(d)
tasks_dic[d] = hietask
except exceptions.IOError:
e = "in file '" + file + \
"', rule or file '" + d + \
"' does not exist"
msg.error(e)
sys.exit(1)
return TaskHierarchical(file, tasks_dic)
try:
return create_hietask(tasks_file)
except exceptions.IOError:
msg.error("file '%s' does not exist.")
sys.exit(1)
def read_hierarchic_tasks(tasks_file):
def read_file_list(file):
fd = open(file, 'r')
lines = fd.readlines()
tasks = TaskList(lines)
fd.close()
return tasks
def read_deps_tokens(line):
# task : dep1 dep2 dep3
taskre = re.compile('[ \t]*:[ \t]*')
try:
task, deps = re.split(taskre, line)
except ValueError:
return None
deps = deps.rstrip('\n')
depsre = re.compile('[\t ]+')
deps = re.split(depsre, deps)
if deps == ['']: deps = []
return task, deps
def read_file_hier(file, tasks_dic):
DEPS_STATE, TASKS_STATE = 0, 1
fd = open(file, 'r')
lines = fd.readlines()
state = DEPS_STATE
tasks = []
task = None
deps = None
for l in lines:
blankre = re.compile('^[ \t]*$')
if blankre.match(l): continue
depsre = re.compile('[a-zA-Z0-9_]*[ \t]*:[ \t]*')
if depsre.match(l):
state = DEPS_STATE
if state == DEPS_STATE:
if task: tasks_dic[task] = deps, tasks
tokens = read_deps_tokens(l)
state = TASKS_STATE
tasks = []
if tokens is None:
task, deps = "all", []
tasks.append(l)
else:
task, deps = tokens
elif state == TASKS_STATE:
tasks.append(l)
if task is not None: tasks_dic[task] = deps, tasks
fd.close()
def create_hietask(file):
tasks_dic = {}
read_file_hier(file, tasks_dic)
if len(tasks_dic) == 0:
msg.error("task file is empty")
sys.exit(1)
for v, (deps, tasks) in tasks_dic.items():
for d in deps:
if tasks_dic.has_key(d): continue
try:
hietask = create_hietask(d)
tasks_dic[d] = hietask
except exceptions.IOError:
e = "in file '" + file + \
"', rule or file '" + d + \
"' does not exist"
msg.error(e)
sys.exit(1)
return TaskHierarchical(file, tasks_dic)
try:
return create_hietask(tasks_file)
except exceptions.IOError:
msg.error("file '%s' does not exist.")
sys.exit(1)