def run_task(): print "Running crawl task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) url = task.data source = urllib.urlopen(url).read() soup = BeautifulSoup(source) links = [] try: for item in soup.find_all('a'): try: links.append(urlparse.urljoin(url, item.get('href'))) except: pass # Not a valid link except: print "Could not fetch any links from html" return res = results.CrawlResult(task.task_id.value, url, links) message = repr(res) driver.sendFrameworkMessage(message) print "Sending status update..." update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print "Sent status update" return
def run(): log.info("Running task %s" % task.task_id.value) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) log.info("Sent: TASK_RUNNING") update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED update.data = "ping" driver.sendStatusUpdate(update) log.info("Sent: TASK_FINISHED")
def run_task(): print "Running task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) time.sleep(1) print "Sending status update..." update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print "Sent status update"
def launchTask(self, driver, task): print "Running task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) Popen("mpd -n -h " + self.ip + " -p " + self.port, shell=True)
def set_done(self): logging.info("Finished Nebula job: %s" % (self.task.task_id.value)) update = mesos_pb2.TaskStatus() update.task_id.value = self.task.task_id.value update.data = str(self.task_data['task_id']) update.state = mesos_pb2.TASK_FINISHED self.driver.sendStatusUpdate(update)
def reply_status(driver, task, status, data=None): update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = status if data is not None: update.data = data driver.sendStatusUpdate(update)
def killTask(self, driver, task_id): logging.debug( "Killing task %s" % task_id.value ) update = mesos_pb2.TaskStatus() update.task_id.value = task_id.value update.state = mesos_pb2.TASK_FINISHED update.data = json.dumps( { 'hostname' : socket.gethostname(), 'task_id' : task_id.value } ) driver.sendStatusUpdate(update)
def run_task(): print "Running task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING update.data = 'data with a \0 byte' driver.sendStatusUpdate(update) # This is where one would perform the requested task. print "Sending status update..." update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED update.data = 'data with a \0 byte' driver.sendStatusUpdate(update) print "Sent status update"
def launchTask(self, driver, task): # Create a thread to run the task. Tasks should always be run in new # threads or processes, rather than inside launchTask itself. def run_task(): task_path = task.task_id.value log("Running task %s" % (task_path)) work_path = os.path.join(self.basedir, rePathClean.sub("", task_path)) update = mesos_pb2.TaskStatus() update.task_id.value = task_path update.state = mesos_pb2.TASK_RUNNING update.data = str(task_path) driver.sendStatusUpdate(update) fail = True try: handle = open(work_path + TARGET_SUFFIX) obj = pickle.loads(handle.read()) handle.close() obj.__manager__ = TargetManager() obj.run() if len(obj.__manager__.child_list): child_dir = os.path.join(work_path + CHILDREN_SUFFIX) if not os.path.exists(child_dir): os.mkdir(child_dir) for name, target in obj.__manager__.child_list: target_path = os.path.join(child_dir, name + TARGET_SUFFIX) handle = open(target_path, "w") handle.write(pickle.dumps(target)) handle.close() if len(obj.__manager__.follow_list): follow_dir = os.path.join(work_path + FOLLOW_SUFFIX) if not os.path.exists(follow_dir): os.mkdir(follow_dir) for name, target in obj.__manager__.follow_list: target_path = os.path.join(follow_dir, name + TARGET_SUFFIX) handle = open(target_path, "w") handle.write(pickle.dumps(target)) handle.close() fail = False except Exception, e: log(str(e)) log("Sending status update...") update = mesos_pb2.TaskStatus() update.task_id.value = task_path if fail: update.state = mesos_pb2.TASK_FAILED else: update.state = mesos_pb2.TASK_FINISHED update.data = str(task_path) driver.sendStatusUpdate(update) log("Sent status update")
def set_running(self): logging.info("Running Nebula job: %s" % (self.task.task_id.value)) nebula_task_id = None try: nebula_task_id = str(self.task_data['task_id']) update = mesos_pb2.TaskStatus() update.task_id.value = self.task.task_id.value update.state = mesos_pb2.TASK_RUNNING update.data = nebula_task_id self.driver.sendStatusUpdate(update)
def run(self): logging.info("Running Nebula task: %s" % (self.task.task_id.value)) nebula_task_id = None try: obj = json.loads(self.task.data) if 'task_type' in obj and obj['task_type'] in task_runner_map: nebula_task_id = str(obj['task_id']) update = mesos_pb2.TaskStatus() update.task_id.value = self.task.task_id.value update.state = mesos_pb2.TASK_RUNNING update.data = nebula_task_id self.driver.sendStatusUpdate(update) cl = task_runner_map[obj['task_type']] inst = cl(obj, self.config) inst.start() outputs = {} for k, v in inst.getOutputs().items(): new_path = os.path.join(self.config.storage_dir, v['uuid']) shutil.move(v['store_path'], new_path) outputs[k] = {'uuid': v['uuid'], 'store_path': new_path} update = mesos_pb2.TaskStatus() update.task_id.value = self.task.task_id.value update.data = json.dumps({ 'task_id': nebula_task_id, 'task_type': obj['task_type'], 'status': "DONE", 'inputs': obj['inputs'], 'outputs': outputs }) update.state = mesos_pb2.TASK_FINISHED self.driver.sendStatusUpdate(update) else: raise Exception("Bad task request") except Exception, e: traceback.print_exc() update = mesos_pb2.TaskStatus() update.task_id.value = self.task.task_id.value update.data = nebula_task_id update.state = mesos_pb2.TASK_FAILED self.driver.sendStatusUpdate(update)
def run_task(): print "Running render task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) url = task.data # 1) Render picture to hash file name. destination = uuid.uuid4().hex + ".png" if call(["phantomjs", "render.js", url, destination]) != 0: print "Could not render " + url return if not self.local: # 2) Upload to s3. remote_destination = "s3://downloads.mesosphere.io/demo/artifacts/" + destination print "Uploading image to " + remote_destination if call(["s3cmd", "put", destination, remote_destination ]) != 0: print "Could not upload " + destination + " to " + remote_destination return else: remote_destination = "file:///" + os.getcwd( ) + "/" + destination # 3) Announce render result to framework. print "Announcing render result" res = results.RenderResult(task.task_id.value, url, remote_destination) message = repr(res) driver.sendFrameworkMessage(message) print "Sending status update for task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print "Sent status update for task %s" % task.task_id.value return
def send_update(self, driver, task_id, state, message=None): update = mesos_pb.TaskStatus() if not isinstance(state, int): raise TypeError('Invalid state type %s, should be int.' % type(state)) if state not in self.MESOS_STATES: raise ValueError('Invalid state: %s' % state) update.state = state update.task_id.value = task_id if message: update.message = str(message) self.log('Updating %s => %s' % (task_id, self.MESOS_STATES[state])) self.log(' Reason: %s' % message) driver.sendStatusUpdate(update)
def run_task(): logging.debug("Mesos Executor: requested task %s", task.task_id.value) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING update.data = str("running discovery task") logging.debug(update.data) driver.sendStatusUpdate(update) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED ## NB: TODO test port availability... update.data = str(dumps(get_telemetry(), indent=4)) ## NB: TODO download tarball/container for service launch # notify scheduler: ready to launch service logging.debug(update.data) driver.sendStatusUpdate(update)
def run_task(): task_path = task.task_id.value log("Running task %s" % (task_path)) work_path = os.path.join(self.basedir, rePathClean.sub("", task_path)) update = mesos_pb2.TaskStatus() update.task_id.value = task_path update.state = mesos_pb2.TASK_RUNNING update.data = str(task_path) driver.sendStatusUpdate(update) fail = True try: handle = open(work_path + TARGET_SUFFIX) obj = pickle.loads(handle.read()) handle.close() obj.__manager__ = TargetManager() obj.run() if len(obj.__manager__.child_list): child_dir = os.path.join(work_path + CHILDREN_SUFFIX) if not os.path.exists(child_dir): os.mkdir(child_dir) for name, target in obj.__manager__.child_list: target_path = os.path.join(child_dir, name + TARGET_SUFFIX) handle = open(target_path, "w") handle.write(pickle.dumps(target)) handle.close() if len(obj.__manager__.follow_list): follow_dir = os.path.join(work_path + FOLLOW_SUFFIX) if not os.path.exists(follow_dir): os.mkdir(follow_dir) for name, target in obj.__manager__.follow_list: target_path = os.path.join(follow_dir, name + TARGET_SUFFIX) handle = open(target_path, "w") handle.write(pickle.dumps(target)) handle.close() fail = False except Exception, e: log(str(e))
def set_error(self): update = mesos_pb2.TaskStatus() update.task_id.value = self.task.task_id.value update.data = str(self.task_data['task_id']) update.state = mesos_pb2.TASK_FAILED self.driver.sendStatusUpdate(update)
def reply_status(driver, task, status): update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = status driver.sendStatusUpdate(update)