def runtask(): update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) data = task.data query_arg = { 'numbers': data, 'sum': sum([int(d) for d in data.split(" ")]) } encode_arg = urllib.urlencode(query_arg) main_url = "http://192.168.12.179:8080/?{param}".format( param=encode_arg) print( urllib2.urlopen(urllib2.Request(main_url, headers=header)).read()) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) pass
def run_task(): print("Running URL crawl task %s" % task.task_id.value) conn = redis.Redis(connection_pool=self.conn_pool) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) url = task.data try: links = url_crawl(url) except CantReadException: error_msg = "Could not read resource at %s" % url update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FAILED update.message = error_msg update.data = url driver.sendStatusUpdate(update) print error_msg return except NotAnyFetchException: error_msg = "Could not fetch any links from html" update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED update.message = error_msg driver.sendStatusUpdate(update) print error_msg return # print("Get these links {}".format(links)) selected_links = [] for link in links: if not conn.hexists(VISITED_LINKS_KEY, link): selected_links.append(link) res = UrlCrawlResult(task.task_id.value, url, links) message = repr(res) driver.sendFrameworkMessage(message) print("Sending status update...") update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print("Sent status update") return
def test_launcher_kill(self): for i in range(self._cluster.num_nodes): task_id, remaining = self._launcher.launch(self._offer) del self._offer.resources[:] self._offer.resources.extend(remaining) assert task_id == "mysos-cluster0-%s" % i tasks = self._driver.method_calls["launchTasks"] assert len(tasks) == self._cluster.num_nodes # No new tasks are launched. assert self._launcher.launch(self._offer)[0] is None assert len(self._driver.method_calls["launchTasks"] ) == self._cluster.num_nodes # All 3 nodes have successfully started. status = mesos_pb2.TaskStatus() status.state = mesos_pb2.TASK_RUNNING # Valid state. status.slave_id.value = self._offer.slave_id.value for i in range(self._cluster.num_nodes): status.task_id.value = "mysos-cluster0-%s" % i self._launcher.status_update(status) deadline( lambda: wait_for_master( get_cluster_path(self._zk_url, self._cluster.name), self. _zk_client), Amount(5, Time.SECONDS)) # The first slave is elected. assert "/mysos/test/cluster0/master/member_0000000000" in self._storage.paths # Two slaves. assert len([ x for x in self._storage.paths.keys() if x.startswith("/mysos/test/cluster0/slaves/member_") ]) == 2 # Kill the cluster. with pytest.raises(MySQLClusterLauncher.PermissionError): self._launcher.kill("wrong_password") self._launcher.kill(self._cluster.password) # Correct password. # All 3 nodes are successfully killed. status = mesos_pb2.TaskStatus() status.state = mesos_pb2.TASK_KILLED status.slave_id.value = self._offer.slave_id.value for i in range(self._cluster.num_nodes): status.task_id.value = "mysos-cluster0-%s" % i self._launcher.status_update(status) assert "/mysos/test/cluster0" not in self._storage.paths # ServerSets removed. assert not self._state_provider.load_cluster_state( "cluster0") # State removed.
def run_task(): update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) print(task.data) time.sleep(30) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update)
def run_task(): print "Running crawl task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) url = task.data source = "" try: source = urllib.urlopen(url).read() except: error_msg = "Could not read resource at %s" % url update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FAILED update.message = error_msg update.data = url driver.sendStatusUpdate(update) print error_msg return soup = BeautifulSoup(source) links = [] try: for item in soup.find_all('a'): try: links.append(urlparse.urljoin(url, item.get('href'))) except: pass # Not a valid link except: print "Could not fetch any links from html" return res = results.CrawlResult(task.task_id.value, url, links) message = repr(res) driver.sendFrameworkMessage(message) print "Sending status update..." update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print "Sent status update" return
def test_launch_cluster_all_nodes_successful(self): for i in range(self._cluster.num_nodes): task_id, remaining = self._launcher.launch(self._offer) del self._offer.resources[:] self._offer.resources.extend(remaining) assert task_id == "mysos-cluster0-%s" % i tasks = self._driver.method_calls["launchTasks"] assert len(tasks) == self._cluster.num_nodes # No new tasks are launched. assert self._launcher.launch(self._offer)[0] is None assert len(self._driver.method_calls["launchTasks"]) == self._cluster.num_nodes # All 3 nodes have successfully started. status = mesos_pb2.TaskStatus() status.state = mesos_pb2.TASK_RUNNING # Valid state. status.slave_id.value = self._offer.slave_id.value for i in range(self._cluster.num_nodes): status.task_id.value = "mysos-cluster0-%s" % i self._launcher.status_update(status) deadline( lambda: wait_for_master( get_cluster_path(self._zk_url, self._cluster.name), self._zk_client), Amount(5, Time.SECONDS)) # The first slave is elected. assert "/mysos/test/cluster0/master/member_0000000000" in self._storage.paths # Two slaves. assert len([x for x in self._storage.paths.keys() if x.startswith( "/mysos/test/cluster0/slaves/member_")]) == 2
def test_invalid_status_update(self): """Launcher raises an exception when an invalid status is received.""" self._cluster.num_nodes = 1 launcher = MySQLClusterLauncher(self._driver, self._cluster, self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml") self._launchers.append(launcher) resources = create_resources(cpus=4, mem=512 * 3, ports=set([10000])) self._offer.resources.extend(resources) task_id, _ = launcher.launch(self._offer) assert task_id == "mysos-cluster0-0" tasks = self._driver.method_calls["launchTasks"] assert len(tasks) == self._cluster.num_nodes status = mesos_pb2.TaskStatus() status.task_id.value = task_id status.state = mesos_pb2.TASK_RUNNING # Valid state. launcher.status_update(status) status.state = mesos_pb2.TASK_FINISHED # An invalid state. with pytest.raises(MySQLClusterLauncher.Error): launcher.status_update(status)
def test_launcher_recovery_corrupted_password(self): # 1. Launch a single instance for a cluster on the running launcher. task_id, remaining = self._launcher.launch(self._offer) del self._offer.resources[:] self._offer.resources.extend(remaining) assert task_id == "mysos-cluster0-0" # The task has successfully started. status = mesos_pb2.TaskStatus() status.state = mesos_pb2.TASK_RUNNING status.slave_id.value = self._offer.slave_id.value status.task_id.value = "mysos-cluster0-0" self._launcher.status_update(status) # 2. Recover the launcher. self._cluster = self._state_provider.load_cluster_state(self._cluster.name) self._cluster.encrypted_password = "******" # The corrupted password causes the launcher constructor to fail. with pytest.raises(ValueError): self._launcher = MySQLClusterLauncher( self._driver, self._cluster, self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", self._scheduler_key, query_interval=Amount(150, Time.MILLISECONDS))
def run_task(): logging.info("Running task: {}".format(task.task_id.value)) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) # TODO: add your code here print(task.data) time.sleep(30) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) logging.info('Task finished, sent final status update.')
def perform_reconciliation(self): """Performs task reconciliation with the Mesos master """ tasks_to_reconcile = {} with self._lock: when = now() # Reconcile all tasks if time threshold has been reached if when > self._last_full_reconciliation + FULL_RECON_THRESHOLD: self._last_full_reconciliation = when for task in self._tasks.values(): tasks_to_reconcile[task.id] = task # Always reconcile rookie tasks and move them to self._tasks # This enables tasks to be quickly reconciled the first time for task in self._rookie_tasks.values(): tasks_to_reconcile[task.id] = task self._tasks[task.id] = task self._rookie_tasks = {} if not tasks_to_reconcile: return logger.info('Reconciling %d task(s)', len(tasks_to_reconcile)) task_statuses = [] for task in tasks_to_reconcile.values(): task_status = mesos_pb2.TaskStatus() task_status.task_id.value = task.id task_status.state = mesos_pb2.TASK_LOST task_statuses.append(task_status) self._driver.reconcileTasks(task_statuses)
def test_scheduler_event_handlers(mocker): sched = mocker.Mock() driver = mocker.Mock() proxy = SchedulerProxy(sched) proxy.executorLost(driver, mesos_pb2.ExecutorInfo(), mesos_pb2.ExecutorID(), 1) proxy.frameworkMessage(driver, mesos_pb2.ExecutorID(), mesos_pb2.SlaveID(), 'message') proxy.offerRescinded(driver, mesos_pb2.OfferID()) proxy.registered(driver, mesos_pb2.FrameworkID(), mesos_pb2.MasterInfo()) proxy.resourceOffers(driver, [mesos_pb2.Offer(), mesos_pb2.Offer()]) proxy.slaveLost(driver, mesos_pb2.SlaveID()) proxy.statusUpdate(driver, mesos_pb2.TaskStatus()) proxy.reregistered(driver, mesos_pb2.MasterInfo()) proxy.error(driver, 'message') proxy.disconnected(driver) sched.on_executor_lost.assert_called_once() sched.on_message.assert_called_once() sched.on_rescinded.assert_called_once() sched.on_registered.assert_called_once() sched.on_offers.assert_called_once() sched.on_slave_lost.assert_called_once() sched.on_update.assert_called_once() sched.on_reregistered.assert_called_once() sched.on_error.assert_called_once() sched.on_disconnected.assert_called_once()
def run_task(): print "Running task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING update.data = "run" driver.sendStatusUpdate(update) cmd = "LD_LIBRARY_PATH=/data/opt/lib:/data/opt/brew/lib " + task.data print cmd print os.system(cmd) update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED update.data = "done" driver.sendStatusUpdate(update)
def run_task(): """ Run a Apache Mesos Task. """ # start openlava services tmp = json.loads(task.data) master_hostname = tmp['master_hostname'] master_ip = tmp['master_ip'] agent_hostname = tmp['agent_hostname'] agent_ip = tmp['agent_ip'] # configure the master. util.add_to_hosts(master_hostname, master_ip) util.add_to_cluster_conf(master_hostname) # tell master we are ready to fire up. update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING update.data = str(agent_hostname) + ':' + str(agent_ip) driver.sendStatusUpdate(update) util.start_lava() # in case I'm idle for a while done. busy = True count = 0 while busy: time.sleep(10) try: if util.njobs_per_host(agent_hostname.strip()) == 0: count += 1 except: # lim not ready... pass if count >= 12: busy = False # tell master we are done here. update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED update.data = str(agent_hostname) + ':' + str(agent_ip) driver.sendStatusUpdate(update) util.stop_lava()
def sendUpdate(taskState, message=''): log.debug("Sending status update ...") status = mesos_pb2.TaskStatus() status.task_id.value = task.task_id.value status.message = message status.state = taskState driver.sendStatusUpdate(status) log.debug("Sent status update")
def run_task(): print "Running task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING update.data = 'data with a \0 byte' driver.sendStatusUpdate(update) # This is where one would perform the requested task. print "Sending status update..." update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED update.data = 'data with a \0 byte' driver.sendStatusUpdate(update) print "Sent status update"
def reply_status(driver, task_id, state, data=None): status = mesos_pb2.TaskStatus() status.task_id.MergeFrom(task_id) status.state = state status.timestamp = time.time() if data is not None: status.data = data driver.sendStatusUpdate(status)
class PythonTaskStatus(PickleMixin, TaskStatus): proto = mesos_pb2.TaskStatus( labels=mesos_pb2.Labels( labels=[mesos_pb2.Label(key='python')])) def __init__(self, data=None, **kwargs): super(PythonTaskStatus, self).__init__(**kwargs) self.data = data
def killTask(self, driver, taskId): update = mesos_pb2.TaskStatus() update.task_id.value = taskId.value update.state = mesos_pb2.TASK_KILLED res_dict = {"message": "Killed on request from the scheduler"} update.data = json.dumps(res_dict) # Send update, then die driver.sendStatusUpdate(update) raise SystemExit(res_dict['message'])
def test_scheduler_metrics(self): scheduler_key = gen_encryption_key() scheduler = MysosScheduler( self._state, self._state_provider, self._framework_user, "./executor.pex", "cmd.sh", self._zk_client, self._zk_url, Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", scheduler_key) RootMetrics().register_observable('scheduler', scheduler) scheduler.registered(self._driver, self._framework_id, object()) sample = RootMetrics().sample() assert sample['scheduler.framework_registered'] == 1 scheduler.create_cluster( "cluster1", "mysql_user", 3, cluster_password='******') sample = RootMetrics().sample() assert sample['scheduler.cluster_count'] == 1 assert sample['scheduler.total_requested_mem_mb'] == DEFAULT_TASK_MEM.as_(Data.MB) * 3 assert sample['scheduler.total_requested_disk_mb'] == DEFAULT_TASK_DISK.as_(Data.MB) * 3 assert sample['scheduler.total_requested_cpus'] == DEFAULT_TASK_CPUS * 3 scheduler.resourceOffers(self._driver, [self._offer]) sample = RootMetrics().sample() assert sample['scheduler.resource_offers'] == 1 assert sample['scheduler.tasks_launched'] == 1 status = mesos_pb2.TaskStatus() status.state = mesos_pb2.TASK_RUNNING status.slave_id.value = self._offer.slave_id.value status.task_id.value = 'mysos-cluster1-0' scheduler.statusUpdate(self._driver, status) status.state = mesos_pb2.TASK_FAILED scheduler.statusUpdate(self._driver, status) sample = RootMetrics().sample() assert sample['scheduler.tasks_failed'] == 1 scheduler.delete_cluster("cluster1", 'test_password') sample = RootMetrics().sample() assert sample['scheduler.cluster_count'] == 0 assert sample['scheduler.total_requested_mem_mb'] == 0 assert sample['scheduler.total_requested_disk_mb'] == 0 assert sample['scheduler.total_requested_cpus'] == 0
def sendUpdate(taskState, wallTime=None, message=''): log.debug('Sending task status update ...') status = mesos_pb2.TaskStatus() status.task_id.value = task.task_id.value status.message = message status.state = taskState if wallTime is not None: status.data = pack('d', wallTime) driver.sendStatusUpdate(status) log.debug('... done sending task status update.')
def get_task_status(self, agent): """ Return a TaskStatus object for this Task. :return: A Mesos TaskStatus instance. """ task_status = mesos_pb2.TaskStatus() task_status.slave_id.value = agent.agent_id task_status.task_id.value = self.task_id task_status.state = self.state return task_status
def collect_cpu_and_memory(): while True: update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value collector = Collector() update.message = collector.collect_info() update.state = mesos_pb2.TASK_RUNNING logger.debug(update.message) driver.sendStatusUpdate(update) time.sleep(30)
def run_task(): print "Running render task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING driver.sendStatusUpdate(update) url = task.data # 1) Render picture to hash file name. destination = uuid.uuid4().hex + ".png" if call(["phantomjs", "render.js", url, destination]) != 0: print "Could not render " + url return if not self.local: # 2) Upload to s3. remote_destination = "s3://downloads.mesosphere.io/demo/artifacts/" + destination print "Uploading image to " + remote_destination if call(["s3cmd", "put", destination, remote_destination]) != 0: print "Could not upload " + destination + " to " + remote_destination return else: remote_destination = "file:///" + os.getcwd() + "/" + destination # 3) Announce render result to framework. print "Announcing render result" res = results.RenderResult( task.task_id.value, url, remote_destination ) message = repr(res) driver.sendFrameworkMessage(message) print "Sending status update for task %s" % task.task_id.value update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print "Sent status update for task %s" % task.task_id.value return
def run_task(): """ Run a Apache Mesos Task. """ # start openlava services tmp = json.loads(task.data) host = tmp['master_host'] ip_addr = tmp['master_ip'] util.add_to_hosts(host, ip_addr) util.add_to_cluster_conf(host) slave_host, slave_ip = util.get_ip() update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_RUNNING update.data = slave_host + ':' + slave_ip driver.sendStatusUpdate(update) util.start_lava() # in case I'm idle for a while done. busy = True count = 0 while busy: time.sleep(10) try: if util.njobs_per_host(slave_host.strip()) == 0: count += 1 except: # lim not ready... pass if count >= 12: busy = False update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED update.data = slave_host + ':' + slave_ip driver.sendStatusUpdate(update) util.stop_lava()
def test_python_task_status_decode(): data = {'arbitrary': 'data', 'lst': [1, 2, 3]} dumped = cloudpickle.dumps(data) proto = mesos_pb2.TaskStatus( data=dumped, labels=mesos_pb2.Labels(labels=[mesos_pb2.Label(key='python')])) status = decode(proto) assert isinstance(status, PythonTaskStatus) assert status['data'] == dumped assert status.data == data proto = mesos_pb2.TaskStatus(labels=mesos_pb2.Labels( labels=[mesos_pb2.Label(key='python')])) status = decode(proto) status.data = data assert isinstance(status, PythonTaskStatus) assert status.data == data assert status['data'] == dumped
def send_update(self, state, message=None, data=None): update = mesos_pb2.TaskStatus() update.task_id.value = self.id update.state = state update.message = message or '' update.data = data or '' if state in [mesos_pb2.TASK_FAILED, mesos_pb2.TASK_ERROR, mesos_pb2.TASK_LOST]: update.healthy = False else: update.healthy = True driver.sendStatusUpdate(update)
def send_update(self, driver, task_id, state, message=None): update = mesos_pb2.TaskStatus() if not isinstance(state, int): raise TypeError('Invalid state type %s, should be int.' % type(state)) if state not in self.STATES_TO_STR: raise ValueError('Invalid state: %s' % state) update.state = state update.task_id.value = task_id if message: update.message = str(message) self.log('Updating %s => %s' % (task_id, self.STATES_TO_STR[state])) self.log(' Reason: %s' % message) driver.sendStatusUpdate(update)
def reply_status(self, driver, proc_id, status, message='', data=tuple()): update = mesos_pb2.TaskStatus() update.task_id.value = str(proc_id) update.slave_id.MergeFrom(self.slave_id) update.timestamp = time.time() update.state = status update.timestamp = time.time() if message: update.message = message if data: update.data = pickle.dumps(data) driver.sendStatusUpdate(update)
def fail(message, executor_driver, task): """ Helper function to send failure status update to the framework. :param message: Message string to send to the framework. :param executor_driver: Executor driver. :param task: Failed task. :return: None """ update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FAILED update.message = message executor_driver.sendStatusUpdate(update)
def test_terminal_status_update(self): """Launcher reacts to terminated task by launching a new one.""" self._cluster.num_nodes = 1 launcher = MySQLClusterLauncher( self._driver, self._cluster, self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(1, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", self._scheduler_key) self._launchers.append(launcher) resources = create_resources( cpus=DEFAULT_TASK_CPUS, mem=DEFAULT_TASK_MEM, disk=DEFAULT_TASK_DISK, ports=set([10000])) self._offer.resources.extend(resources) task_id, _ = launcher.launch(self._offer) assert task_id == "mysos-cluster0-0" launched = self._driver.method_calls["launchTasks"] assert len(launched) == self._cluster.num_nodes status = mesos_pb2.TaskStatus() status.task_id.value = task_id status.state = mesos_pb2.TASK_RUNNING launcher.status_update(status) assert len(launcher._cluster.running_tasks) == 1 status.state = mesos_pb2.TASK_LOST launcher.status_update(status) assert len(launcher._cluster.running_tasks) == 0 task_id, _ = launcher.launch(self._offer) assert task_id == "mysos-cluster0-1" launched = self._driver.method_calls["launchTasks"] # One task is relaunched to make up for the lost one. assert len(launched) == self._cluster.num_nodes + 1