def runtask():
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
data = task.data
query_arg = {
'numbers': data,
'sum': sum([int(d) for d in data.split(" ")])
}
encode_arg = urllib.urlencode(query_arg)
main_url = "http://192.168.12.179:8080/?{param}".format(
param=encode_arg)
print(
urllib2.urlopen(urllib2.Request(main_url,
headers=header)).read())
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
pass
def run_task():
print("Running URL crawl task %s" % task.task_id.value)
conn = redis.Redis(connection_pool=self.conn_pool)
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
url = task.data
try:
links = url_crawl(url)
except CantReadException:
error_msg = "Could not read resource at %s" % url
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FAILED
update.message = error_msg
update.data = url
driver.sendStatusUpdate(update)
print error_msg
return
except NotAnyFetchException:
error_msg = "Could not fetch any links from html"
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.message = error_msg
driver.sendStatusUpdate(update)
print error_msg
return
# print("Get these links {}".format(links))
selected_links = []
for link in links:
if not conn.hexists(VISITED_LINKS_KEY, link):
selected_links.append(link)
res = UrlCrawlResult(task.task_id.value, url, links)
message = repr(res)
driver.sendFrameworkMessage(message)
print("Sending status update...")
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print("Sent status update")
return
def test_launcher_kill(self):
for i in range(self._cluster.num_nodes):
task_id, remaining = self._launcher.launch(self._offer)
del self._offer.resources[:]
self._offer.resources.extend(remaining)
assert task_id == "mysos-cluster0-%s" % i
tasks = self._driver.method_calls["launchTasks"]
assert len(tasks) == self._cluster.num_nodes
# No new tasks are launched.
assert self._launcher.launch(self._offer)[0] is None
assert len(self._driver.method_calls["launchTasks"]
) == self._cluster.num_nodes
# All 3 nodes have successfully started.
status = mesos_pb2.TaskStatus()
status.state = mesos_pb2.TASK_RUNNING # Valid state.
status.slave_id.value = self._offer.slave_id.value
for i in range(self._cluster.num_nodes):
status.task_id.value = "mysos-cluster0-%s" % i
self._launcher.status_update(status)
deadline(
lambda: wait_for_master(
get_cluster_path(self._zk_url, self._cluster.name), self.
_zk_client), Amount(5, Time.SECONDS))
# The first slave is elected.
assert "/mysos/test/cluster0/master/member_0000000000" in self._storage.paths
# Two slaves.
assert len([
x for x in self._storage.paths.keys()
if x.startswith("/mysos/test/cluster0/slaves/member_")
]) == 2
# Kill the cluster.
with pytest.raises(MySQLClusterLauncher.PermissionError):
self._launcher.kill("wrong_password")
self._launcher.kill(self._cluster.password) # Correct password.
# All 3 nodes are successfully killed.
status = mesos_pb2.TaskStatus()
status.state = mesos_pb2.TASK_KILLED
status.slave_id.value = self._offer.slave_id.value
for i in range(self._cluster.num_nodes):
status.task_id.value = "mysos-cluster0-%s" % i
self._launcher.status_update(status)
assert "/mysos/test/cluster0" not in self._storage.paths # ServerSets removed.
assert not self._state_provider.load_cluster_state(
"cluster0") # State removed.
def run_task():
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
print(task.data)
time.sleep(30)
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
def run_task():
print "Running crawl task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
url = task.data
source = ""
try:
source = urllib.urlopen(url).read()
except:
error_msg = "Could not read resource at %s" % url
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FAILED
update.message = error_msg
update.data = url
driver.sendStatusUpdate(update)
print error_msg
return
soup = BeautifulSoup(source)
links = []
try:
for item in soup.find_all('a'):
try:
links.append(urlparse.urljoin(url, item.get('href')))
except:
pass # Not a valid link
except:
print "Could not fetch any links from html"
return
res = results.CrawlResult(task.task_id.value, url, links)
message = repr(res)
driver.sendFrameworkMessage(message)
print "Sending status update..."
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update"
return
def test_launch_cluster_all_nodes_successful(self):
for i in range(self._cluster.num_nodes):
task_id, remaining = self._launcher.launch(self._offer)
del self._offer.resources[:]
self._offer.resources.extend(remaining)
assert task_id == "mysos-cluster0-%s" % i
tasks = self._driver.method_calls["launchTasks"]
assert len(tasks) == self._cluster.num_nodes
# No new tasks are launched.
assert self._launcher.launch(self._offer)[0] is None
assert len(self._driver.method_calls["launchTasks"]) == self._cluster.num_nodes
# All 3 nodes have successfully started.
status = mesos_pb2.TaskStatus()
status.state = mesos_pb2.TASK_RUNNING # Valid state.
status.slave_id.value = self._offer.slave_id.value
for i in range(self._cluster.num_nodes):
status.task_id.value = "mysos-cluster0-%s" % i
self._launcher.status_update(status)
deadline(
lambda: wait_for_master(
get_cluster_path(self._zk_url, self._cluster.name),
self._zk_client),
Amount(5, Time.SECONDS))
# The first slave is elected.
assert "/mysos/test/cluster0/master/member_0000000000" in self._storage.paths
# Two slaves.
assert len([x for x in self._storage.paths.keys() if x.startswith(
"/mysos/test/cluster0/slaves/member_")]) == 2
def test_invalid_status_update(self):
"""Launcher raises an exception when an invalid status is received."""
self._cluster.num_nodes = 1
launcher = MySQLClusterLauncher(self._driver, self._cluster,
self._state_provider, self._zk_url,
self._zk_client, self._framework_user,
"./executor.pex", "cmd.sh",
Amount(5, Time.SECONDS),
"/etc/mysos/admin_keyfile.yml")
self._launchers.append(launcher)
resources = create_resources(cpus=4, mem=512 * 3, ports=set([10000]))
self._offer.resources.extend(resources)
task_id, _ = launcher.launch(self._offer)
assert task_id == "mysos-cluster0-0"
tasks = self._driver.method_calls["launchTasks"]
assert len(tasks) == self._cluster.num_nodes
status = mesos_pb2.TaskStatus()
status.task_id.value = task_id
status.state = mesos_pb2.TASK_RUNNING # Valid state.
launcher.status_update(status)
status.state = mesos_pb2.TASK_FINISHED # An invalid state.
with pytest.raises(MySQLClusterLauncher.Error):
launcher.status_update(status)
def test_launcher_recovery_corrupted_password(self):
# 1. Launch a single instance for a cluster on the running launcher.
task_id, remaining = self._launcher.launch(self._offer)
del self._offer.resources[:]
self._offer.resources.extend(remaining)
assert task_id == "mysos-cluster0-0"
# The task has successfully started.
status = mesos_pb2.TaskStatus()
status.state = mesos_pb2.TASK_RUNNING
status.slave_id.value = self._offer.slave_id.value
status.task_id.value = "mysos-cluster0-0"
self._launcher.status_update(status)
# 2. Recover the launcher.
self._cluster = self._state_provider.load_cluster_state(self._cluster.name)
self._cluster.encrypted_password = "******"
# The corrupted password causes the launcher constructor to fail.
with pytest.raises(ValueError):
self._launcher = MySQLClusterLauncher(
self._driver,
self._cluster,
self._state_provider,
self._zk_url,
self._zk_client,
self._framework_user,
"./executor.pex",
"cmd.sh",
Amount(5, Time.SECONDS),
"/etc/mysos/admin_keyfile.yml",
self._scheduler_key,
query_interval=Amount(150, Time.MILLISECONDS))
def run_task():
logging.info("Running task: {}".format(task.task_id.value))
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
# TODO: add your code here
print(task.data)
time.sleep(30)
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
logging.info('Task finished, sent final status update.')
def perform_reconciliation(self):
"""Performs task reconciliation with the Mesos master
"""
tasks_to_reconcile = {}
with self._lock:
when = now()
# Reconcile all tasks if time threshold has been reached
if when > self._last_full_reconciliation + FULL_RECON_THRESHOLD:
self._last_full_reconciliation = when
for task in self._tasks.values():
tasks_to_reconcile[task.id] = task
# Always reconcile rookie tasks and move them to self._tasks
# This enables tasks to be quickly reconciled the first time
for task in self._rookie_tasks.values():
tasks_to_reconcile[task.id] = task
self._tasks[task.id] = task
self._rookie_tasks = {}
if not tasks_to_reconcile:
return
logger.info('Reconciling %d task(s)', len(tasks_to_reconcile))
task_statuses = []
for task in tasks_to_reconcile.values():
task_status = mesos_pb2.TaskStatus()
task_status.task_id.value = task.id
task_status.state = mesos_pb2.TASK_LOST
task_statuses.append(task_status)
self._driver.reconcileTasks(task_statuses)
def test_scheduler_event_handlers(mocker):
sched = mocker.Mock()
driver = mocker.Mock()
proxy = SchedulerProxy(sched)
proxy.executorLost(driver, mesos_pb2.ExecutorInfo(),
mesos_pb2.ExecutorID(), 1)
proxy.frameworkMessage(driver, mesos_pb2.ExecutorID(),
mesos_pb2.SlaveID(), 'message')
proxy.offerRescinded(driver, mesos_pb2.OfferID())
proxy.registered(driver, mesos_pb2.FrameworkID(), mesos_pb2.MasterInfo())
proxy.resourceOffers(driver, [mesos_pb2.Offer(), mesos_pb2.Offer()])
proxy.slaveLost(driver, mesos_pb2.SlaveID())
proxy.statusUpdate(driver, mesos_pb2.TaskStatus())
proxy.reregistered(driver, mesos_pb2.MasterInfo())
proxy.error(driver, 'message')
proxy.disconnected(driver)
sched.on_executor_lost.assert_called_once()
sched.on_message.assert_called_once()
sched.on_rescinded.assert_called_once()
sched.on_registered.assert_called_once()
sched.on_offers.assert_called_once()
sched.on_slave_lost.assert_called_once()
sched.on_update.assert_called_once()
sched.on_reregistered.assert_called_once()
sched.on_error.assert_called_once()
sched.on_disconnected.assert_called_once()
def run_task():
print "Running task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = "run"
driver.sendStatusUpdate(update)
cmd = "LD_LIBRARY_PATH=/data/opt/lib:/data/opt/brew/lib " + task.data
print cmd
print os.system(cmd)
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = "done"
driver.sendStatusUpdate(update)
def run_task():
"""
Run a Apache Mesos Task.
"""
# start openlava services
tmp = json.loads(task.data)
master_hostname = tmp['master_hostname']
master_ip = tmp['master_ip']
agent_hostname = tmp['agent_hostname']
agent_ip = tmp['agent_ip']
# configure the master.
util.add_to_hosts(master_hostname, master_ip)
util.add_to_cluster_conf(master_hostname)
# tell master we are ready to fire up.
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = str(agent_hostname) + ':' + str(agent_ip)
driver.sendStatusUpdate(update)
util.start_lava()
# in case I'm idle for a while done.
busy = True
count = 0
while busy:
time.sleep(10)
try:
if util.njobs_per_host(agent_hostname.strip()) == 0:
count += 1
except:
# lim not ready...
pass
if count >= 12:
busy = False
# tell master we are done here.
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = str(agent_hostname) + ':' + str(agent_ip)
driver.sendStatusUpdate(update)
util.stop_lava()
def sendUpdate(taskState, message=''):
log.debug("Sending status update ...")
status = mesos_pb2.TaskStatus()
status.task_id.value = task.task_id.value
status.message = message
status.state = taskState
driver.sendStatusUpdate(status)
log.debug("Sent status update")
def run_task():
print "Running task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = 'data with a \0 byte'
driver.sendStatusUpdate(update)
# This is where one would perform the requested task.
print "Sending status update..."
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = 'data with a \0 byte'
driver.sendStatusUpdate(update)
print "Sent status update"
def reply_status(driver, task_id, state, data=None):
status = mesos_pb2.TaskStatus()
status.task_id.MergeFrom(task_id)
status.state = state
status.timestamp = time.time()
if data is not None:
status.data = data
driver.sendStatusUpdate(status)
class PythonTaskStatus(PickleMixin, TaskStatus):
proto = mesos_pb2.TaskStatus(
labels=mesos_pb2.Labels(
labels=[mesos_pb2.Label(key='python')]))
def __init__(self, data=None, **kwargs):
super(PythonTaskStatus, self).__init__(**kwargs)
self.data = data
def killTask(self, driver, taskId):
update = mesos_pb2.TaskStatus()
update.task_id.value = taskId.value
update.state = mesos_pb2.TASK_KILLED
res_dict = {"message": "Killed on request from the scheduler"}
update.data = json.dumps(res_dict)
# Send update, then die
driver.sendStatusUpdate(update)
raise SystemExit(res_dict['message'])
def test_scheduler_metrics(self):
scheduler_key = gen_encryption_key()
scheduler = MysosScheduler(
self._state,
self._state_provider,
self._framework_user,
"./executor.pex",
"cmd.sh",
self._zk_client,
self._zk_url,
Amount(5, Time.SECONDS),
"/etc/mysos/admin_keyfile.yml",
scheduler_key)
RootMetrics().register_observable('scheduler', scheduler)
scheduler.registered(self._driver, self._framework_id, object())
sample = RootMetrics().sample()
assert sample['scheduler.framework_registered'] == 1
scheduler.create_cluster(
"cluster1", "mysql_user", 3, cluster_password='******')
sample = RootMetrics().sample()
assert sample['scheduler.cluster_count'] == 1
assert sample['scheduler.total_requested_mem_mb'] == DEFAULT_TASK_MEM.as_(Data.MB) * 3
assert sample['scheduler.total_requested_disk_mb'] == DEFAULT_TASK_DISK.as_(Data.MB) * 3
assert sample['scheduler.total_requested_cpus'] == DEFAULT_TASK_CPUS * 3
scheduler.resourceOffers(self._driver, [self._offer])
sample = RootMetrics().sample()
assert sample['scheduler.resource_offers'] == 1
assert sample['scheduler.tasks_launched'] == 1
status = mesos_pb2.TaskStatus()
status.state = mesos_pb2.TASK_RUNNING
status.slave_id.value = self._offer.slave_id.value
status.task_id.value = 'mysos-cluster1-0'
scheduler.statusUpdate(self._driver, status)
status.state = mesos_pb2.TASK_FAILED
scheduler.statusUpdate(self._driver, status)
sample = RootMetrics().sample()
assert sample['scheduler.tasks_failed'] == 1
scheduler.delete_cluster("cluster1", 'test_password')
sample = RootMetrics().sample()
assert sample['scheduler.cluster_count'] == 0
assert sample['scheduler.total_requested_mem_mb'] == 0
assert sample['scheduler.total_requested_disk_mb'] == 0
assert sample['scheduler.total_requested_cpus'] == 0
def sendUpdate(taskState, wallTime=None, message=''):
log.debug('Sending task status update ...')
status = mesos_pb2.TaskStatus()
status.task_id.value = task.task_id.value
status.message = message
status.state = taskState
if wallTime is not None:
status.data = pack('d', wallTime)
driver.sendStatusUpdate(status)
log.debug('... done sending task status update.')
def get_task_status(self, agent):
"""
Return a TaskStatus object for this Task.
:return: A Mesos TaskStatus instance.
"""
task_status = mesos_pb2.TaskStatus()
task_status.slave_id.value = agent.agent_id
task_status.task_id.value = self.task_id
task_status.state = self.state
return task_status
def collect_cpu_and_memory():
while True:
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
collector = Collector()
update.message = collector.collect_info()
update.state = mesos_pb2.TASK_RUNNING
logger.debug(update.message)
driver.sendStatusUpdate(update)
time.sleep(30)
def run_task():
print "Running render task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
url = task.data
# 1) Render picture to hash file name.
destination = uuid.uuid4().hex + ".png"
if call(["phantomjs", "render.js", url, destination]) != 0:
print "Could not render " + url
return
if not self.local:
# 2) Upload to s3.
remote_destination = "s3://downloads.mesosphere.io/demo/artifacts/" + destination
print "Uploading image to " + remote_destination
if call(["s3cmd", "put", destination, remote_destination]) != 0:
print "Could not upload " + destination + " to " + remote_destination
return
else:
remote_destination = "file:///" + os.getcwd() + "/" + destination
# 3) Announce render result to framework.
print "Announcing render result"
res = results.RenderResult(
task.task_id.value,
url,
remote_destination
)
message = repr(res)
driver.sendFrameworkMessage(message)
print "Sending status update for task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update for task %s" % task.task_id.value
return
def run_task():
"""
Run a Apache Mesos Task.
"""
# start openlava services
tmp = json.loads(task.data)
host = tmp['master_host']
ip_addr = tmp['master_ip']
util.add_to_hosts(host, ip_addr)
util.add_to_cluster_conf(host)
slave_host, slave_ip = util.get_ip()
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = slave_host + ':' + slave_ip
driver.sendStatusUpdate(update)
util.start_lava()
# in case I'm idle for a while done.
busy = True
count = 0
while busy:
time.sleep(10)
try:
if util.njobs_per_host(slave_host.strip()) == 0:
count += 1
except:
# lim not ready...
pass
if count >= 12:
busy = False
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = slave_host + ':' + slave_ip
driver.sendStatusUpdate(update)
util.stop_lava()
def test_python_task_status_decode():
data = {'arbitrary': 'data', 'lst': [1, 2, 3]}
dumped = cloudpickle.dumps(data)
proto = mesos_pb2.TaskStatus(
data=dumped,
labels=mesos_pb2.Labels(labels=[mesos_pb2.Label(key='python')]))
status = decode(proto)
assert isinstance(status, PythonTaskStatus)
assert status['data'] == dumped
assert status.data == data
proto = mesos_pb2.TaskStatus(labels=mesos_pb2.Labels(
labels=[mesos_pb2.Label(key='python')]))
status = decode(proto)
status.data = data
assert isinstance(status, PythonTaskStatus)
assert status.data == data
assert status['data'] == dumped
def send_update(self, state, message=None, data=None):
update = mesos_pb2.TaskStatus()
update.task_id.value = self.id
update.state = state
update.message = message or ''
update.data = data or ''
if state in [mesos_pb2.TASK_FAILED,
mesos_pb2.TASK_ERROR,
mesos_pb2.TASK_LOST]:
update.healthy = False
else:
update.healthy = True
driver.sendStatusUpdate(update)
def send_update(self, driver, task_id, state, message=None):
update = mesos_pb2.TaskStatus()
if not isinstance(state, int):
raise TypeError('Invalid state type %s, should be int.' %
type(state))
if state not in self.STATES_TO_STR:
raise ValueError('Invalid state: %s' % state)
update.state = state
update.task_id.value = task_id
if message:
update.message = str(message)
self.log('Updating %s => %s' % (task_id, self.STATES_TO_STR[state]))
self.log(' Reason: %s' % message)
driver.sendStatusUpdate(update)
def reply_status(self, driver, proc_id, status, message='', data=tuple()):
update = mesos_pb2.TaskStatus()
update.task_id.value = str(proc_id)
update.slave_id.MergeFrom(self.slave_id)
update.timestamp = time.time()
update.state = status
update.timestamp = time.time()
if message:
update.message = message
if data:
update.data = pickle.dumps(data)
driver.sendStatusUpdate(update)
def fail(message, executor_driver, task):
"""
Helper function to send failure status update to the framework.
:param message: Message string to send to the framework.
:param executor_driver: Executor driver.
:param task: Failed task.
:return: None
"""
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FAILED
update.message = message
executor_driver.sendStatusUpdate(update)
def test_terminal_status_update(self):
"""Launcher reacts to terminated task by launching a new one."""
self._cluster.num_nodes = 1
launcher = MySQLClusterLauncher(
self._driver,
self._cluster,
self._state_provider,
self._zk_url,
self._zk_client,
self._framework_user,
"./executor.pex",
"cmd.sh",
Amount(1, Time.SECONDS),
"/etc/mysos/admin_keyfile.yml",
self._scheduler_key)
self._launchers.append(launcher)
resources = create_resources(
cpus=DEFAULT_TASK_CPUS,
mem=DEFAULT_TASK_MEM,
disk=DEFAULT_TASK_DISK,
ports=set([10000]))
self._offer.resources.extend(resources)
task_id, _ = launcher.launch(self._offer)
assert task_id == "mysos-cluster0-0"
launched = self._driver.method_calls["launchTasks"]
assert len(launched) == self._cluster.num_nodes
status = mesos_pb2.TaskStatus()
status.task_id.value = task_id
status.state = mesos_pb2.TASK_RUNNING
launcher.status_update(status)
assert len(launcher._cluster.running_tasks) == 1
status.state = mesos_pb2.TASK_LOST
launcher.status_update(status)
assert len(launcher._cluster.running_tasks) == 0
task_id, _ = launcher.launch(self._offer)
assert task_id == "mysos-cluster0-1"
launched = self._driver.method_calls["launchTasks"]
# One task is relaunched to make up for the lost one.
assert len(launched) == self._cluster.num_nodes + 1
