def main(): parser = optparse.OptionParser() parser.add_option('-c', '--command', help = 'Command to execute.', default = 'echo "Frame $WRANGLER_PADFRAME on $HOSTNAME at `date`"') parser.add_option('-n', '--name', help = 'Name of Render Job.', default = 'No Name') parser.add_option('-v', '--verbose', help = 'Print verbose output.', action='store_true', default = False) parser.add_option('-s', '--start', help = 'Start frame of render.', default = 1, type=int) parser.add_option('-e', '--end', help = 'End frame of render.', default = 1, type=int) parser.add_option('-p', '--priority', help = "The job's priority. [0 - 1000]", default = 500, type=int) client = LassoClient() opts, args = parser.parse_args() if opts.verbose: print 'Creating Job' job_data = {} job_data['generator'] = 'RenderJob' job_data['command'] = opts.command job_data['start'] = opts.start job_data['end'] = opts.end job_data['priority'] = opts.priority job_data['owner'] = os.environ['USER'] job_data['name'] = opts.name job_data['env'] = os.environ.copy() if opts.verbose: print("""Adding Job: Name: %s Command: %s Start: %d End: %d Priority: %d""" % (opts.name, opts.command, opts.start, opts.end, opts.priority)) id = client.lasso_submit_job(job_data) print 'Job [%d] was added to queue.' % id
def _setup(self): WranglerServer._setup(self) self.client = LassoClient() self.running_tasks = {} self.max_tasks = self.config.getint('cattle', 'max-tasks') self._state = self.ASLEEP #Setup Handlers self._handles.append(self._handle_metrics) self._handles.append(self._handle_pulse) #self._handles.append(self._handle_monitor) #Setup Timeouts self._register_timeout('metrics', self.config.getfloat('cattle', 'metric-frequency')) self._register_timeout('task-request', self.config.getfloat('cattle', 'idle')) self._register_timeout('pulse', 15.0) self._no_tasks = False #Thread Locks self.num_thread_lock = thread.allocate_lock() #Register XMLRPC calls self.server.register_function(self.monitor_connect, 'monitor_connect') self.server.register_function(self.monitor_disconnect, 'monitor_disconnect') self.server.register_function(self.monitor_probe, 'monitor_probe') self.server.register_function(self.monitor_start, 'monitor_start') self.server.register_function(self.kill_task, 'kill_task') self.server.register_function(self.sleep, 'sleep') self.server.register_function(self.wake, 'wake') self.server.register_function(self.task_list, 'task_list') self.server.register_function(self.state, 'state') #Connect to lasso. self.cattle = connect_cattle(self.hostname) self.wake()
def cmd_help(): client = LassoClient() print client.help()
class CattleServer(WranglerServer): ASLEEP = 0 AWAKE = 1 def __init__(self): self.configure() hostname = info.hostname() port = self.config.getint('cattle', 'port') WranglerServer.__init__(self, hostname, port, 'wrangler.cattle') def _setup(self): WranglerServer._setup(self) self.client = LassoClient() self.running_tasks = {} self.max_tasks = self.config.getint('cattle', 'max-tasks') self._state = self.ASLEEP #Setup Handlers self._handles.append(self._handle_metrics) self._handles.append(self._handle_pulse) #self._handles.append(self._handle_monitor) #Setup Timeouts self._register_timeout('metrics', self.config.getfloat('cattle', 'metric-frequency')) self._register_timeout('task-request', self.config.getfloat('cattle', 'idle')) self._register_timeout('pulse', 15.0) self._no_tasks = False #Thread Locks self.num_thread_lock = thread.allocate_lock() #Register XMLRPC calls self.server.register_function(self.monitor_connect, 'monitor_connect') self.server.register_function(self.monitor_disconnect, 'monitor_disconnect') self.server.register_function(self.monitor_probe, 'monitor_probe') self.server.register_function(self.monitor_start, 'monitor_start') self.server.register_function(self.kill_task, 'kill_task') self.server.register_function(self.sleep, 'sleep') self.server.register_function(self.wake, 'wake') self.server.register_function(self.task_list, 'task_list') self.server.register_function(self.state, 'state') #Connect to lasso. self.cattle = connect_cattle(self.hostname) self.wake() def configure(self): self.config = config_cattle() def full(self): self.num_thread_lock.acquire() v = bool(len(self.running_tasks) >= self.max_tasks) self.num_thread_lock.release() return v def shutdown(self): WranglerServer.shutdown(self) disconnect_cattle(self.hostname) def sleep(self): self.info("Cattle is going to sleep.") if self._state == self.AWAKE: self._state = self.ASLEEP sleep_cattle(self.hostname) return self._state def wake(self): self.info("Cattle is waking up.") if self._state == self.ASLEEP: self._state = self.AWAKE wake_cattle(self.hostname) return self._state def task_list(self): return map(str, self.running_tasks.keys()) def state(self): return self._state def request_task(self): self.debug('Requesting task from server.') taskid = self.client.next_task(self.hostname) if taskid > 0: self.info('Task %d was received from server.' % taskid) db = Session() task = db.query(Task).filter_by(id=taskid).first() db.expunge(task) db.close() if task: self.num_thread_lock.acquire() self.running_tasks[taskid] = task self.num_thread_lock.release() self._pre_task(task) if not self._run_task(task): self._clean_up_task(task) self._no_tasks = False else: self.debug('No task was received from server.') self._no_tasks = True def monitor_connect(self, task_id): self.debug('Monitor for task %d has connected.' % task_id) db = Session() task = self.running_tasks[task_id] db.add(task) db.add(task.log) task_data = {} task_data['command'] = task.run_command tries = 0 while tries < 3: try: task_data['gid'] = pwd.getpwnam(task.job.owner)[3] task_data['uid'] = pwd.getpwnam(task.job.owner)[2] break except KeyError: tries += 1 task_data['stdout_file_path'] = task.log._stdout_file_path() task_data['stderr_file_path'] = task.log._stderr_file_path() db.close() return task_data def monitor_start(self, task_id, pid): self.debug('Task %d started process %d' % (task_id, pid)) self.num_thread_lock.acquire() task = self.running_tasks[task_id] task.pid = pid self.num_thread_lock.release() return pid def monitor_probe(self, task_id, probes): self.debug("Logging probe for %d" % task_id) task = self.running_tasks[task_id] db = Session() probe = TaskProbe() db.add(probe) cattle = self.cattle probe.task_id = task_id probe.memory = probes['memory'] probe.pcpu = probes['pcpu'] probe.pid = probes['pid'] #probe.cattle_id = task.log.cattle_id probe.time = datetime.datetime.now() probe.probes = probes probe.cattle_id = cattle.id probe.task_log_id = task.log.id db.commit() probe_id = probe.id db.expunge(probe) db.close() return probe_id def kill_task(self, task_id): try: task = self.running_tasks[task_id] except KeyError: return False try: pid = task.pid except AttributeError: return False try: os.kill(pid,signal.SIGTERM) except OSError: os.kill(pid, signal.SIGKILL) return True def monitor_disconnect(self, task_id, run_time, return_code): self.debug('Monitor for task %d had disconnected.' % task_id) self._post_task(self.running_tasks[task_id], run_time, return_code) self._clean_up_task(self.running_tasks[task_id]) return task_id def _handle_pulse(self): if self._timeout('pulse'): self.debug('Pulsing lasso server.') self.client.pulse(self.hostname) def _handle_metrics(self): if self._timeout('metrics'): self.debug('Logging metrics') metrics = {} metrics['hostname'] = info.hostname() metrics['time'] = time.time() metrics['load_avg'] = info.load_avg() metrics['memory'] = info.memory() metrics['running'] = self.running_tasks.keys() update_metrics(info.hostname(), metrics) def _handle_main(self): if self._state == self.ASLEEP: return if not self.full(): if self._no_tasks: if self._timeout('task-request'): self.request_task() else: self.request_task() def _pre_task(self, task): self.debug('Preprocessing task %d' % task.id) # Create final task command. task.run_command = task.command env = os.environ.copy() env.update(task.env) for k, v in env.copy().iteritems(): env[str(k)] = str(v) task.run_env = env task.owner = task.job.owner # Create task log. task.log = create_task_log(task, self.cattle) def _run_task(self, task): self.debug('Executing task %d' % task.id) #Start Process cattle_monitor_path = os.path.join(os.path.dirname(__file__), 'monitor.py') command_call = '%s %d' % (cattle_monitor_path, task.id) try: task.start_time = time.time() task.proc = subprocess.Popen(command_call, env = task.run_env, shell = True) except OSError, msg: self.error(msg) update_task_log(task.log, -1, -1) return False return True