示例#1
0
def remake(non_empty_job_list):  
    '''Remake the selected targets (equivalent to clean and make). '''
    
    non_empty_job_list = list(non_empty_job_list) 
        
    
    from compmake.ui.console import ask_question
    if get_compmake_status() == compmake_status_interactive: 
        question = "Should I clean and remake %d jobs? [y/n] " % \
            len(non_empty_job_list)
        answer = ask_question(question)
        if not answer:
            info('Not cleaned.')
            return
        
    for job in non_empty_job_list:
        mark_remake(job)
    
    manager = ManagerLocal()
    manager.add_targets(non_empty_job_list)
    manager.process()
    
    if manager.failed:
        return RET_CODE_JOB_FAILED
    else:
        return 0
示例#2
0
def reload(module):
    ''' Reloads a module.
    
        Usage::
        
            reload module=my_module
    
    '''

    if module.startswith('compmake'):
        try:
            dave = pwd.getpwuid(os.getuid())[0]
        except:
            dave = 'Dave'
        user_error("I'm sorry, %s. I'm afraid I can't do that." % dave)
        return
        
    try:     
        # otherwise import("A.B") returns A instead of A.B
        m = __import__(module, fromlist=['dummy'])
    except Exception as e:
        raise UserError('Cannot find module "%s": %s.' % (module, e))
        
    try: 
        imp.reload(m)
    except Exception as e:
        raise UserError('Obtained this exception while reloading the module:'
                        ' %s' % e)
    
    info('Reloaded module "%s".' % module)
    
示例#3
0
def display_stats(job_list):
    
    states_order = [Cache.NOT_STARTED, Cache.IN_PROGRESS,
              Cache.MORE_REQUESTED, Cache.FAILED,
              Cache.DONE]
    # initialize counters to 0
    states2count = dict(map(lambda x: (x, 0), states_order))

    function2state2count = {}
    total = 0
    
    for job_id in job_list:
        
        cache = get_job_cache(job_id)   
        states2count[cache.state] += 1
        total += 1
        
        function_id = get_job(job_id).command_desc
        # initialize record if not present
        if not function_id in function2state2count:
            function2state2count[function_id] = dict(map(lambda x: (x, 0), states_order) + 
                                                     [('all', 0)])
        # update
        function2state2count[function_id][cache.state] += 1
        function2state2count[function_id]['all'] += 1
        
        if total == 100: # XXX: use standard method
            info("Loading a large number of jobs...")
    
    print("Found %s jobs in total. Summary by state:" % total)
        
    for state in states_order:
        desc = "%30s" % Cache.state2desc[state]
        # colorize output
        desc = colored(desc, **state2color[state])

        num = states2count[state]
        if num > 0:
            print("%s: %5d" % (desc, num))
          
    print("Summary by function:")

    for function_id, function_stats in function2state2count.items():
        ndone = function_stats[Cache.DONE]
        nfailed = function_stats[Cache.FAILED]
        nrest = function_stats['all'] - ndone - nfailed
        failed_s = "%5d failed" % nfailed
        if nfailed > 0:
            failed_s = colored(failed_s, color='red')
        s = "%5d done, %s, %5d to do." % (ndone, failed_s, nrest)
        
        print(" %30s(): %s" % (function_id, s)) 
示例#4
0
    def process(self):
        ''' Start processing jobs. '''
        
        # precompute job priorities
        #print "Computing priorities..."
        self.priorities = compute_priorities(self.all_targets)
        #print "... done"
        
        if not self.todo:
            info('Nothing to do.')
            return True
        
        self.process_init()
        
        try:
            while self.todo:
                assert self.ready_todo or self.processing 
                assert not self.failed.intersection(self.todo)
        
                self.publish_progress()
                self.instance_some_jobs()
                self.publish_progress()
                if self.ready_todo and not self.processing:
                    publish('manager-failed', reason='No resources.',
                        targets=self.targets, done=self.done,
                        todo=self.todo, failed=self.failed, ready=self.ready_todo,
                        processing=self.processing, all_targets=self.all_targets)
        
                    raise CompmakeException('Cannot find computing resources, giving up.') 
                
                self.publish_progress()
                self.loop_until_something_finishes()
               
            self.process_finished()
        
            publish('manager-succeeded',
                targets=self.targets, done=self.done, all_targets=self.all_targets,
                todo=self.todo, failed=self.failed, ready=self.ready_todo,
                processing=self.processing)

            return True

        except JobInterrupted:
            # XXX I'm getting confused
            raise KeyboardInterrupt
示例#5
0
def more(non_empty_job_list, loop=1):
    '''Makes more of the selected targets. '''
    
    non_empty_job_list = list(non_empty_job_list)
    
    
    for x in range(int(loop)):
        if loop > 1:
            info("------- more: iteration %d --- " % x) 

        for job in non_empty_job_list:
            mark_more(job)
            
        manager = ManagerLocal()
        manager.add_targets(non_empty_job_list, more=True)
        manager.process()
        
        if manager.failed:
            return RET_CODE_JOB_FAILED

    return 0
示例#6
0
文件: ui.py 项目: welinder/compmake
def config(args):
    ''' Get/set configuration parameters.

Call like:

    @> config  <switch>  <value>
         
Without arguments, shows all configuration switches.
 '''    
    if not args:
        # show
        show_config(sys.stdout)
        return
        
    name = args.pop(0)
    if not args:
        if not name in config_switches:
            raise UserError("I don't know the switch '%s'." % name)
        info('config %s %s' % (name, compmake_config.__dict__[name]))
        return
        
    set_config_from_strings(name, args)
示例#7
0
def clean(job_list):
    '''Cleans the result of the selected computation \
(or everything is nothing specified). '''

    job_list = list(job_list)
    
    if not job_list:
        job_list = list(all_jobs())
        
    if not job_list:
        return 
    
    from compmake.ui.console import ask_question
    
    if get_compmake_status() == compmake_status_interactive: 
        question = "Should I clean %d jobs? [y/n] " % len(job_list)
        answer = ask_question(question)
        if not answer:
            info('Not cleaned.')
            return
        
    for job_id in job_list:
        clean_target(job_id)
示例#8
0
def parmore(non_empty_job_list, loop=1):
    '''Parallel equivalent of "more". '''
    
    non_empty_job_list = list(non_empty_job_list)
    
    for job in non_empty_job_list:
        mark_more(job)
        
    for x in range(int(loop)):
        if loop > 1:
            info("------- parmore: iteration %d --- " % x) 

        for job in non_empty_job_list:
            mark_more(job)

        manager = MultiprocessingManager()
        manager.add_targets(non_empty_job_list, more=True)
        manager.process()
        
        if manager.failed:
            return RET_CODE_JOB_FAILED

    return 0
示例#9
0
def clustmore(non_empty_job_list, loop=1):
    '''Cluster equivalent of "more".

       Note: you should use the Redis backend to use multiprocessing.
 ''' 
    cluster_conf = compmake_config.cluster_conf #@UndefinedVariable
    hosts = parse_yaml_configuration(open(cluster_conf))
    
    for x in range(int(loop)):
        if loop > 1:
            info("------- more: iteration %d --- " % x) 

        for job in non_empty_job_list:
            mark_more(job)
            
            manager = ClusterManager(hosts)
        manager.add_targets(non_empty_job_list, more=True)
        manager.process()
        
        if manager.failed:
            return RET_CODE_JOB_FAILED
        
    return 0
示例#10
0
 def my_host_failed(self, host):
     self.failed_hosts.add(host)
     while host in self.hosts_ready:
         self.hosts_ready.remove(host)
     info('Host %s failed, removing from stack (failed now %s)' % 
              (host, self.failed_hosts))
示例#11
0
def set_namespace(n):
    info('Using namespace %r.' % n) 
    sys.modules["compmake.jobs.storage"].namespace = n