示例#1
0
文件: grid.py 项目: Solvi/pyhrf
def run_grid(mode, hosts_list, keytype, tasks, timeslot, brokenfile,
             logfile, user=None, passwd=None, time_limit=86400):
    if 0:
        print 'run_grid ...'
        print 'mode:', mode
        print 'hosts_list:', hosts_list
        print 'keytype:', keytype
        print 'tasks:', tasks
        print 'timeslot:', timeslot
        print 'brokenfile:', brokenfile
        print 'logfile:', logfile
        print 'user:'******'passwd:', passwd
        print 'time_limit:', time_limit
        print ''
        print ''
    hm = HostsManager(hosts_list)
    if len(hm._available_list) == 0:
        msg.error("All hosts down :(")
        return
    if tasks is None : return
        #print 'run_grid ...'
        #print 'tasks:', tasks
    brokenfd = open(brokenfile, 'w')
    log = open(logfile, 'w')
    user = User(user, passwd, keytype)
    args = (timeslot, user, tasks, hm, log, brokenfd, time_limit)
    if mode == 'dispatch': tm = DispatchedTasksManager(*args)
    elif mode == 'repeat': tm = RepeatedTasksManager(*args)
    elif mode in ['hie', 'hierarchic']:
        tm = HierarchicalTasksManager(*args)
    tm.start()
    brokenfd.close()
    log.close()
示例#2
0
文件: grid.py 项目: Solvi/pyhrf
def read_hosts(hosts):
    if hosts.find(',') == -1:
        try:
            fd = open(hosts, 'r')
        except exceptions.IOError:
            msg.error("host file '%s' can not be opened" % hosts)
        else:
            hosts = ' '.join(fd.readlines()).replace('\n', ' ')
            fd.close()
    hosts_list = re.split('[\s,]+', hosts)
    if hosts_list[-1] == '': del hosts_list[-1]
    return hosts_list
示例#3
0
def read_hosts(hosts):
    if hosts.find(',') == -1:
        try:
            fd = open(hosts, 'r')
        except exceptions.IOError:
            msg.error("host file '%s' can not be opened" % hosts)
        else:
            hosts = ' '.join(fd.readlines()).replace('\n', ' ')
            fd.close()
    hosts_list = re.split('[\s,]+', hosts)
    if hosts_list[-1] == '': del hosts_list[-1]
    return hosts_list
示例#4
0
def parse_options(parser):
    morehelpmap = {
        'hosts': hosts_help,
        'tasks': tasks_help,
        'mode': mode_help,
        'broken': broken_help,
        'log': log_help,
        'timeslot': timeslot_help
    }
    (options, args) = parser.parse_args(sys.argv)
    error = False
    error_msg = []

    if options.morehelp != None:
        try:
            morehelpmap[options.morehelp](sys.argv[0])
            sys.exit(1)
        except KeyError:
            msg.error("No help for '%s'.\n" % options.morehelp)
            msg.write_list(
                [' ', ('*', 'green'), ' List of available full helps :\n   '])
            for k in morehelpmap.keys():
                msg.write('%s, ' % k)
            msg.write('\n')
            sys.exit(1)
    mandatory_options = {'hosts': options.hosts}
    timeslot = read_timeslot(options.timeslot)
    if None in mandatory_options.values():
        opt = [n for n, o in mandatory_options.items() if o == None]
        error_msg.append('missing options : %s' % opt)
        error = True
    if not options.mode in ["dispatch", "repeat", "hie", "hierarchical"]:
        error_msg.append("'%s' : unknown task manager mode." % \
                            options.mode)
        error = True
    if error:
        parser.print_help()
        for m in error_msg:
            msg.error(m)
        sys.exit(1)
    hosts_list = read_hosts(options.hosts)
    if options.tasks:
        tasks_list = read_tasks(options.tasks, options.mode)
    else:
        tasks_list = None
    return options, timeslot, hosts_list, tasks_list
示例#5
0
文件: grid.py 项目: Solvi/pyhrf
 def create_hietask(file):
     tasks_dic = {}
     read_file_hier(file, tasks_dic)
     if len(tasks_dic) == 0:
         msg.error("task file is empty")
         sys.exit(1)
     for v, (deps, tasks) in tasks_dic.items():
         for d in deps:
             if tasks_dic.has_key(d): continue
             try:
                 hietask = create_hietask(d)
                 tasks_dic[d] = hietask
             except exceptions.IOError:
                 e = "in file '" + file + \
                     "', rule or file '" + d + \
                     "' does not exist"
                 msg.error(e)
                 sys.exit(1)
     return TaskHierarchical(file, tasks_dic)
示例#6
0
 def create_hietask(file):
     tasks_dic = {}
     read_file_hier(file, tasks_dic)
     if len(tasks_dic) == 0:
         msg.error("task file is empty")
         sys.exit(1)
     for v, (deps, tasks) in tasks_dic.items():
         for d in deps:
             if tasks_dic.has_key(d): continue
             try:
                 hietask = create_hietask(d)
                 tasks_dic[d] = hietask
             except exceptions.IOError:
                 e = "in file '" + file + \
                     "', rule or file '" + d + \
                     "' does not exist"
                 msg.error(e)
                 sys.exit(1)
     return TaskHierarchical(file, tasks_dic)
示例#7
0
def run_grid(mode,
             hosts_list,
             keytype,
             tasks,
             timeslot,
             brokenfile=None,
             logfile=None,
             user=None,
             passwd=None,
             time_limit=86400):
    if 0:
        print 'run_grid ...'
        print 'mode:', mode
        print 'hosts_list:', hosts_list
        print 'keytype:', keytype
        print 'tasks:', tasks
        print 'timeslot:', timeslot
        print 'brokenfile:', brokenfile
        print 'logfile:', logfile
        print 'user:'******'passwd:', passwd
        print 'time_limit:', time_limit
        print ''
        print ''
    hm = HostsManager(hosts_list)
    if len(hm._available_list) == 0:
        msg.error("All hosts down :(")
        return
    if tasks is None: return
    #print 'run_grid ...'
    #print 'tasks:', tasks
    brokenfd = open(brokenfile or os.devnull, 'w')
    log = open(logfile or os.devnull, 'w')
    user = User(user, passwd, keytype)
    args = (timeslot, user, tasks, hm, log, brokenfd, time_limit)
    if mode == 'dispatch': tm = DispatchedTasksManager(*args)
    elif mode == 'repeat': tm = RepeatedTasksManager(*args)
    elif mode in ['hie', 'hierarchic']:
        tm = HierarchicalTasksManager(*args)
    tm.start()
    brokenfd.close()
    log.close()
示例#8
0
文件: grid.py 项目: Solvi/pyhrf
def parse_options(parser):
    morehelpmap = {'hosts' : hosts_help, 'tasks' : tasks_help,
        'mode' : mode_help, 'broken' : broken_help, 'log' : log_help,
        'timeslot' : timeslot_help}
    (options, args) = parser.parse_args(sys.argv)
    error = False
    error_msg = []

    if options.morehelp != None:
        try:
            morehelpmap[options.morehelp](sys.argv[0])
            sys.exit(1)
        except KeyError:
            msg.error("No help for '%s'.\n" % options.morehelp)
            msg.write_list([' ', ('*', 'green'),
                ' List of available full helps :\n   '])
            for k in morehelpmap.keys(): msg.write('%s, ' % k)
            msg.write('\n')
            sys.exit(1)
    mandatory_options = {'hosts' : options.hosts}
    timeslot = read_timeslot(options.timeslot)
    if None in mandatory_options.values():
        opt = [n for n, o in mandatory_options.items() if o == None]
        error_msg.append('missing options : %s' % opt)
        error = True
    if not options.mode in ["dispatch", "repeat", "hie", "hierarchical"]:
        error_msg.append("'%s' : unknown task manager mode." % \
                            options.mode)
        error = True
    if error:
        parser.print_help()
        for m in error_msg: msg.error(m)
        sys.exit(1)
    hosts_list = read_hosts(options.hosts)
    if options.tasks:
        tasks_list = read_tasks(options.tasks, options.mode)
    else:    tasks_list = None
    return options, timeslot, hosts_list, tasks_list
示例#9
0
文件: grid.py 项目: Solvi/pyhrf
def read_timeslot(timeslot):
    if timeslot in ( '-', 'allday' ):
        return TimeSlot(0, 86400) # [0:24h]
    elif timeslot == 'morning':
        return TimeSlot(0, 43200) # [0:12h]
    elif timeslot == 'afternoon':
        return TimeSlot(43200, 86400) # [12h:24h]
    elif timeslot == 'midi':
        return TimeSlot(43200, 50400) # [12h:14h]
    elif timeslot == 'night':
        # [0:8h] + [20h:24h] = [20h:8h]
        return TimeSlotList([TimeSlot(0, 28800),
                TimeSlot(72000, 86400)])

        timeslot_list = timeslot.split(',')

    tsl = []
    for one_timeslot in timeslot_list:
        try:
            t1, t2 = one_timeslot.split('-')
        except ValueError:
            msg.error("'%s' : bad timeslot format" % one_timeslot)
            sys.exit(1)
        seconds = []
        for t in [t1, t2]:
            try:
                h, m, s = t.split(':')
            except ValueError:
                msg.error("'%s' : bad hour format" % t)
                sys.exit(1)
            h, m, s = int(h), int(m), int(s)
            seconds.append(s + 60 * (m + 60 * h))
        tsl.append(TimeSlot(seconds[0], seconds[1]))
    if len(timeslot_list) > 1:
        ts = TimeSlotList(tsl)
    else:    ts = tsl[0]
    return ts
示例#10
0
def read_timeslot(timeslot):
    if timeslot in ('-', 'allday'):
        return TimeSlot(0, 86400)  # [0:24h]
    elif timeslot == 'morning':
        return TimeSlot(0, 43200)  # [0:12h]
    elif timeslot == 'afternoon':
        return TimeSlot(43200, 86400)  # [12h:24h]
    elif timeslot == 'midi':
        return TimeSlot(43200, 50400)  # [12h:14h]
    elif timeslot == 'night':
        # [0:8h] + [20h:24h] = [20h:8h]
        return TimeSlotList([TimeSlot(0, 28800), TimeSlot(72000, 86400)])

        timeslot_list = timeslot.split(',')

    tsl = []
    for one_timeslot in timeslot_list:
        try:
            t1, t2 = one_timeslot.split('-')
        except ValueError:
            msg.error("'%s' : bad timeslot format" % one_timeslot)
            sys.exit(1)
        seconds = []
        for t in [t1, t2]:
            try:
                h, m, s = t.split(':')
            except ValueError:
                msg.error("'%s' : bad hour format" % t)
                sys.exit(1)
            h, m, s = int(h), int(m), int(s)
            seconds.append(s + 60 * (m + 60 * h))
        tsl.append(TimeSlot(seconds[0], seconds[1]))
    if len(timeslot_list) > 1:
        ts = TimeSlotList(tsl)
    else:
        ts = tsl[0]
    return ts
示例#11
0
文件: grid.py 项目: Solvi/pyhrf
def read_hierarchic_tasks(tasks_file):
    def read_file_list(file):
        fd = open(file, 'r')
        lines = fd.readlines()
        tasks = TaskList(lines)
        fd.close()
        return tasks

    def read_deps_tokens(line):
        # task : dep1 dep2 dep3
        taskre = re.compile('[ \t]*:[ \t]*')
        try:
            task, deps = re.split(taskre, line)
        except ValueError:
            return None
        deps = deps.rstrip('\n')
        depsre = re.compile('[\t ]+')
        deps = re.split(depsre, deps)
        if deps == ['']: deps = []
        return task, deps

    def read_file_hier(file, tasks_dic):
        DEPS_STATE, TASKS_STATE = 0, 1
        fd = open(file, 'r')
        lines = fd.readlines()
        state = DEPS_STATE
        tasks = []
        task = None
        deps = None
        for l in lines:
            blankre = re.compile('^[ \t]*$')
            if blankre.match(l): continue
            depsre = re.compile('[a-zA-Z0-9_]*[ \t]*:[ \t]*')
            if depsre.match(l):
                state = DEPS_STATE
            if state == DEPS_STATE:
                if task: tasks_dic[task] = deps, tasks
                tokens = read_deps_tokens(l)
                state = TASKS_STATE
                tasks = []
                if tokens is None:
                    task, deps = "all", []
                    tasks.append(l)
                else:    task, deps = tokens
            elif state == TASKS_STATE:
                tasks.append(l)
        if task is not None: tasks_dic[task] = deps, tasks
        fd.close()

    def create_hietask(file):
        tasks_dic = {}
        read_file_hier(file, tasks_dic)
        if len(tasks_dic) == 0:
            msg.error("task file is empty")
            sys.exit(1)
        for v, (deps, tasks) in tasks_dic.items():
            for d in deps:
                if tasks_dic.has_key(d): continue
                try:
                    hietask = create_hietask(d)
                    tasks_dic[d] = hietask
                except exceptions.IOError:
                    e = "in file '" + file + \
                        "', rule or file '" + d + \
                        "' does not exist"
                    msg.error(e)
                    sys.exit(1)
        return TaskHierarchical(file, tasks_dic)

    try:
        return create_hietask(tasks_file)
    except exceptions.IOError:
        msg.error("file '%s' does not exist.")
        sys.exit(1)
示例#12
0
def read_hierarchic_tasks(tasks_file):
    def read_file_list(file):
        fd = open(file, 'r')
        lines = fd.readlines()
        tasks = TaskList(lines)
        fd.close()
        return tasks

    def read_deps_tokens(line):
        # task : dep1 dep2 dep3
        taskre = re.compile('[ \t]*:[ \t]*')
        try:
            task, deps = re.split(taskre, line)
        except ValueError:
            return None
        deps = deps.rstrip('\n')
        depsre = re.compile('[\t ]+')
        deps = re.split(depsre, deps)
        if deps == ['']: deps = []
        return task, deps

    def read_file_hier(file, tasks_dic):
        DEPS_STATE, TASKS_STATE = 0, 1
        fd = open(file, 'r')
        lines = fd.readlines()
        state = DEPS_STATE
        tasks = []
        task = None
        deps = None
        for l in lines:
            blankre = re.compile('^[ \t]*$')
            if blankre.match(l): continue
            depsre = re.compile('[a-zA-Z0-9_]*[ \t]*:[ \t]*')
            if depsre.match(l):
                state = DEPS_STATE
            if state == DEPS_STATE:
                if task: tasks_dic[task] = deps, tasks
                tokens = read_deps_tokens(l)
                state = TASKS_STATE
                tasks = []
                if tokens is None:
                    task, deps = "all", []
                    tasks.append(l)
                else:
                    task, deps = tokens
            elif state == TASKS_STATE:
                tasks.append(l)
        if task is not None: tasks_dic[task] = deps, tasks
        fd.close()

    def create_hietask(file):
        tasks_dic = {}
        read_file_hier(file, tasks_dic)
        if len(tasks_dic) == 0:
            msg.error("task file is empty")
            sys.exit(1)
        for v, (deps, tasks) in tasks_dic.items():
            for d in deps:
                if tasks_dic.has_key(d): continue
                try:
                    hietask = create_hietask(d)
                    tasks_dic[d] = hietask
                except exceptions.IOError:
                    e = "in file '" + file + \
                        "', rule or file '" + d + \
                        "' does not exist"
                    msg.error(e)
                    sys.exit(1)
        return TaskHierarchical(file, tasks_dic)

    try:
        return create_hietask(tasks_file)
    except exceptions.IOError:
        msg.error("file '%s' does not exist.")
        sys.exit(1)