Ejemplo n.º 1
0
def main():
    parser = optparse.OptionParser()
    parser.add_option('-c', '--command',
                      help = 'Command to execute.',
                      default = 'echo "Frame $WRANGLER_PADFRAME on $HOSTNAME at `date`"')
    parser.add_option('-n', '--name',
                      help = 'Name of Render Job.',
                      default = 'No Name')
    parser.add_option('-v', '--verbose',
                      help = 'Print verbose output.',
                      action='store_true',
                      default = False)
    parser.add_option('-s', '--start',
                      help = 'Start frame of render.',
                      default = 1,
                      type=int)
    parser.add_option('-e', '--end',
                      help = 'End frame of render.',
                      default = 1,
                      type=int)
    parser.add_option('-p', '--priority',
                      help = "The job's priority. [0 - 1000]",
                      default = 500,
                      type=int)
    
    client = LassoClient()
    opts, args = parser.parse_args()
    if opts.verbose:
        print 'Creating Job'
    job_data = {}
    job_data['generator'] = 'RenderJob'
    job_data['command'] = opts.command
    job_data['start'] = opts.start
    job_data['end'] = opts.end
    job_data['priority'] = opts.priority
    job_data['owner'] = os.environ['USER']
    job_data['name'] = opts.name
    job_data['env'] = os.environ.copy()
    
    if opts.verbose:
        print("""Adding Job:
Name:     %s
Command:  %s
Start:    %d
End:      %d
Priority: %d""" % (opts.name, opts.command, opts.start, opts.end, opts.priority))
    id = client.lasso_submit_job(job_data)
    
    print 'Job [%d] was added to queue.' % id
Ejemplo n.º 2
0
    def _setup(self):
        WranglerServer._setup(self)

        self.client = LassoClient()
        self.running_tasks = {}
        self.max_tasks = self.config.getint('cattle', 'max-tasks')

        self._state = self.ASLEEP

        #Setup Handlers
        self._handles.append(self._handle_metrics)
        self._handles.append(self._handle_pulse)
        #self._handles.append(self._handle_monitor)

        #Setup Timeouts
        self._register_timeout('metrics', self.config.getfloat('cattle', 'metric-frequency'))
        self._register_timeout('task-request', self.config.getfloat('cattle', 'idle'))
        self._register_timeout('pulse', 15.0)
        self._no_tasks = False

        #Thread Locks
        self.num_thread_lock = thread.allocate_lock()

        #Register XMLRPC calls
        self.server.register_function(self.monitor_connect, 'monitor_connect')
        self.server.register_function(self.monitor_disconnect, 'monitor_disconnect')
        self.server.register_function(self.monitor_probe, 'monitor_probe')
        self.server.register_function(self.monitor_start, 'monitor_start')
        self.server.register_function(self.kill_task, 'kill_task')
        self.server.register_function(self.sleep, 'sleep')
        self.server.register_function(self.wake, 'wake')
        self.server.register_function(self.task_list, 'task_list')
        self.server.register_function(self.state, 'state')

        #Connect to lasso.
        self.cattle = connect_cattle(self.hostname)
        self.wake()
Ejemplo n.º 3
0
def cmd_help():
    client = LassoClient()
    print client.help()
Ejemplo n.º 4
0
class CattleServer(WranglerServer):
    ASLEEP = 0
    AWAKE = 1
    
    def __init__(self):
        self.configure()
        hostname = info.hostname()
        port = self.config.getint('cattle', 'port')
        WranglerServer.__init__(self, hostname, port, 'wrangler.cattle')

    def _setup(self):
        WranglerServer._setup(self)

        self.client = LassoClient()
        self.running_tasks = {}
        self.max_tasks = self.config.getint('cattle', 'max-tasks')

        self._state = self.ASLEEP

        #Setup Handlers
        self._handles.append(self._handle_metrics)
        self._handles.append(self._handle_pulse)
        #self._handles.append(self._handle_monitor)

        #Setup Timeouts
        self._register_timeout('metrics', self.config.getfloat('cattle', 'metric-frequency'))
        self._register_timeout('task-request', self.config.getfloat('cattle', 'idle'))
        self._register_timeout('pulse', 15.0)
        self._no_tasks = False

        #Thread Locks
        self.num_thread_lock = thread.allocate_lock()

        #Register XMLRPC calls
        self.server.register_function(self.monitor_connect, 'monitor_connect')
        self.server.register_function(self.monitor_disconnect, 'monitor_disconnect')
        self.server.register_function(self.monitor_probe, 'monitor_probe')
        self.server.register_function(self.monitor_start, 'monitor_start')
        self.server.register_function(self.kill_task, 'kill_task')
        self.server.register_function(self.sleep, 'sleep')
        self.server.register_function(self.wake, 'wake')
        self.server.register_function(self.task_list, 'task_list')
        self.server.register_function(self.state, 'state')

        #Connect to lasso.
        self.cattle = connect_cattle(self.hostname)
        self.wake()

    def configure(self):
        self.config = config_cattle()

    def full(self):
        self.num_thread_lock.acquire()
        v = bool(len(self.running_tasks) >= self.max_tasks)
        self.num_thread_lock.release()
        return v

    def shutdown(self):
        WranglerServer.shutdown(self)
        disconnect_cattle(self.hostname)

    def sleep(self):
        self.info("Cattle is going to sleep.")
        if self._state == self.AWAKE:
            self._state = self.ASLEEP
            sleep_cattle(self.hostname)
        return self._state

    def wake(self):
        self.info("Cattle is waking up.")
        if self._state == self.ASLEEP:
            self._state = self.AWAKE
            wake_cattle(self.hostname)
        return self._state

    def task_list(self):
        return map(str, self.running_tasks.keys())

    def state(self):
        return self._state

    def request_task(self):
        self.debug('Requesting task from server.')
        taskid = self.client.next_task(self.hostname)
        if taskid > 0:
            self.info('Task %d was received from server.' % taskid)
            db = Session()
            task = db.query(Task).filter_by(id=taskid).first()
            db.expunge(task)
            db.close()
            if task:
                self.num_thread_lock.acquire()
                self.running_tasks[taskid] = task
                self.num_thread_lock.release()
                self._pre_task(task)
                if not self._run_task(task):
                    self._clean_up_task(task)
                self._no_tasks = False
        else:
            self.debug('No task was received from server.')
            self._no_tasks = True

    def monitor_connect(self, task_id):
        self.debug('Monitor for task %d has connected.' % task_id)
        db = Session()
        task = self.running_tasks[task_id]
        db.add(task)
        db.add(task.log)
        task_data = {}
        task_data['command'] = task.run_command
        tries = 0
        while tries < 3:
            try:
                task_data['gid'] = pwd.getpwnam(task.job.owner)[3]
                task_data['uid'] = pwd.getpwnam(task.job.owner)[2]
                break
            except KeyError:
                tries += 1
        task_data['stdout_file_path'] = task.log._stdout_file_path()
        task_data['stderr_file_path'] = task.log._stderr_file_path()
        db.close()
        return task_data

    def monitor_start(self, task_id, pid):
        self.debug('Task %d started process %d' % (task_id, pid))
        self.num_thread_lock.acquire()
        task = self.running_tasks[task_id]
        task.pid = pid
        self.num_thread_lock.release()
        return pid

    def monitor_probe(self, task_id, probes):
        self.debug("Logging probe for %d" % task_id)
        task = self.running_tasks[task_id]
        db = Session()
        probe = TaskProbe()
        db.add(probe)
        cattle = self.cattle
        probe.task_id = task_id
        probe.memory = probes['memory']
        probe.pcpu = probes['pcpu']
        probe.pid = probes['pid']
        #probe.cattle_id = task.log.cattle_id
        probe.time = datetime.datetime.now()
        probe.probes = probes
        probe.cattle_id = cattle.id
        probe.task_log_id = task.log.id
        db.commit()
        probe_id = probe.id
        db.expunge(probe)
        db.close()
        return probe_id

    def kill_task(self, task_id):
        try:
            task = self.running_tasks[task_id]
        except KeyError:
            return False
        try:
            pid = task.pid
        except AttributeError:
            return False
        try:
            os.kill(pid,signal.SIGTERM)
        except OSError:
            os.kill(pid, signal.SIGKILL)
        return True

    def monitor_disconnect(self, task_id, run_time, return_code):
        self.debug('Monitor for task %d had disconnected.' % task_id)
        self._post_task(self.running_tasks[task_id], run_time, return_code)
        self._clean_up_task(self.running_tasks[task_id])
        return task_id

    def _handle_pulse(self):
        if self._timeout('pulse'):
            self.debug('Pulsing lasso server.')
            self.client.pulse(self.hostname)

    def _handle_metrics(self):
        if self._timeout('metrics'):
            self.debug('Logging metrics')
            metrics = {}
            metrics['hostname'] = info.hostname()
            metrics['time'] = time.time()
            metrics['load_avg'] = info.load_avg()
            metrics['memory'] = info.memory()
            metrics['running'] = self.running_tasks.keys()
            update_metrics(info.hostname(), metrics)

    def _handle_main(self):
        if self._state == self.ASLEEP:
            return
        if not self.full():
            if self._no_tasks:
                if self._timeout('task-request'):
                    self.request_task()
            else:
                self.request_task()

    def _pre_task(self, task):
        self.debug('Preprocessing task %d' % task.id)

        # Create final task command.
        task.run_command = task.command
        env = os.environ.copy()
        env.update(task.env)
        for k, v in env.copy().iteritems():
            env[str(k)] = str(v)
        task.run_env = env
        task.owner = task.job.owner
        # Create task log.
        task.log = create_task_log(task, self.cattle)

    def _run_task(self, task):
        self.debug('Executing task %d' % task.id)
        #Start Process
        cattle_monitor_path = os.path.join(os.path.dirname(__file__), 'monitor.py')
        command_call = '%s %d' % (cattle_monitor_path, task.id)
        try:
            task.start_time = time.time()
            task.proc = subprocess.Popen(command_call,
                                env = task.run_env,
                                shell = True)
        except OSError, msg:
            self.error(msg)
            update_task_log(task.log, -1, -1)
            return False
        return True