def run_task():
print "Running crawl task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
url = task.data
source = urllib.urlopen(url).read()
soup = BeautifulSoup(source)
links = []
try:
for item in soup.find_all('a'):
try:
links.append(urlparse.urljoin(url, item.get('href')))
except:
pass # Not a valid link
except:
print "Could not fetch any links from html"
return
res = results.CrawlResult(task.task_id.value, url, links)
message = repr(res)
driver.sendFrameworkMessage(message)
print "Sending status update..."
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update"
return
def run():
log.info("Running task %s" % task.task_id.value)
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
log.info("Sent: TASK_RUNNING")
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = "ping"
driver.sendStatusUpdate(update)
log.info("Sent: TASK_FINISHED")
def run_task():
print "Running task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
time.sleep(1)
print "Sending status update..."
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update"
def launchTask(self, driver, task):
print "Running task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
Popen("mpd -n -h " + self.ip + " -p " + self.port, shell=True)
def set_done(self):
logging.info("Finished Nebula job: %s" % (self.task.task_id.value))
update = mesos_pb2.TaskStatus()
update.task_id.value = self.task.task_id.value
update.data = str(self.task_data['task_id'])
update.state = mesos_pb2.TASK_FINISHED
self.driver.sendStatusUpdate(update)
def reply_status(driver, task, status, data=None):
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = status
if data is not None:
update.data = data
driver.sendStatusUpdate(update)
def killTask(self, driver, task_id):
logging.debug( "Killing task %s" % task_id.value )
update = mesos_pb2.TaskStatus()
update.task_id.value = task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = json.dumps( { 'hostname' : socket.gethostname(), 'task_id' : task_id.value } )
driver.sendStatusUpdate(update)
def run_task():
print "Running task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = 'data with a \0 byte'
driver.sendStatusUpdate(update)
# This is where one would perform the requested task.
print "Sending status update..."
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = 'data with a \0 byte'
driver.sendStatusUpdate(update)
print "Sent status update"
def launchTask(self, driver, task):
# Create a thread to run the task. Tasks should always be run in new
# threads or processes, rather than inside launchTask itself.
def run_task():
task_path = task.task_id.value
log("Running task %s" % (task_path))
work_path = os.path.join(self.basedir, rePathClean.sub("", task_path))
update = mesos_pb2.TaskStatus()
update.task_id.value = task_path
update.state = mesos_pb2.TASK_RUNNING
update.data = str(task_path)
driver.sendStatusUpdate(update)
fail = True
try:
handle = open(work_path + TARGET_SUFFIX)
obj = pickle.loads(handle.read())
handle.close()
obj.__manager__ = TargetManager()
obj.run()
if len(obj.__manager__.child_list):
child_dir = os.path.join(work_path + CHILDREN_SUFFIX)
if not os.path.exists(child_dir):
os.mkdir(child_dir)
for name, target in obj.__manager__.child_list:
target_path = os.path.join(child_dir, name + TARGET_SUFFIX)
handle = open(target_path, "w")
handle.write(pickle.dumps(target))
handle.close()
if len(obj.__manager__.follow_list):
follow_dir = os.path.join(work_path + FOLLOW_SUFFIX)
if not os.path.exists(follow_dir):
os.mkdir(follow_dir)
for name, target in obj.__manager__.follow_list:
target_path = os.path.join(follow_dir, name + TARGET_SUFFIX)
handle = open(target_path, "w")
handle.write(pickle.dumps(target))
handle.close()
fail = False
except Exception, e:
log(str(e))
log("Sending status update...")
update = mesos_pb2.TaskStatus()
update.task_id.value = task_path
if fail:
update.state = mesos_pb2.TASK_FAILED
else:
update.state = mesos_pb2.TASK_FINISHED
update.data = str(task_path)
driver.sendStatusUpdate(update)
log("Sent status update")
def set_running(self):
logging.info("Running Nebula job: %s" % (self.task.task_id.value))
nebula_task_id = None
try:
nebula_task_id = str(self.task_data['task_id'])
update = mesos_pb2.TaskStatus()
update.task_id.value = self.task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = nebula_task_id
self.driver.sendStatusUpdate(update)
def run(self):
logging.info("Running Nebula task: %s" % (self.task.task_id.value))
nebula_task_id = None
try:
obj = json.loads(self.task.data)
if 'task_type' in obj and obj['task_type'] in task_runner_map:
nebula_task_id = str(obj['task_id'])
update = mesos_pb2.TaskStatus()
update.task_id.value = self.task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = nebula_task_id
self.driver.sendStatusUpdate(update)
cl = task_runner_map[obj['task_type']]
inst = cl(obj, self.config)
inst.start()
outputs = {}
for k, v in inst.getOutputs().items():
new_path = os.path.join(self.config.storage_dir, v['uuid'])
shutil.move(v['store_path'], new_path)
outputs[k] = {'uuid': v['uuid'], 'store_path': new_path}
update = mesos_pb2.TaskStatus()
update.task_id.value = self.task.task_id.value
update.data = json.dumps({
'task_id': nebula_task_id,
'task_type': obj['task_type'],
'status': "DONE",
'inputs': obj['inputs'],
'outputs': outputs
})
update.state = mesos_pb2.TASK_FINISHED
self.driver.sendStatusUpdate(update)
else:
raise Exception("Bad task request")
except Exception, e:
traceback.print_exc()
update = mesos_pb2.TaskStatus()
update.task_id.value = self.task.task_id.value
update.data = nebula_task_id
update.state = mesos_pb2.TASK_FAILED
self.driver.sendStatusUpdate(update)
def run_task():
print "Running render task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
url = task.data
# 1) Render picture to hash file name.
destination = uuid.uuid4().hex + ".png"
if call(["phantomjs", "render.js", url, destination]) != 0:
print "Could not render " + url
return
if not self.local:
# 2) Upload to s3.
remote_destination = "s3://downloads.mesosphere.io/demo/artifacts/" + destination
print "Uploading image to " + remote_destination
if call(["s3cmd", "put", destination, remote_destination
]) != 0:
print "Could not upload " + destination + " to " + remote_destination
return
else:
remote_destination = "file:///" + os.getcwd(
) + "/" + destination
# 3) Announce render result to framework.
print "Announcing render result"
res = results.RenderResult(task.task_id.value, url,
remote_destination)
message = repr(res)
driver.sendFrameworkMessage(message)
print "Sending status update for task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update for task %s" % task.task_id.value
return
def send_update(self, driver, task_id, state, message=None):
update = mesos_pb.TaskStatus()
if not isinstance(state, int):
raise TypeError('Invalid state type %s, should be int.' % type(state))
if state not in self.MESOS_STATES:
raise ValueError('Invalid state: %s' % state)
update.state = state
update.task_id.value = task_id
if message:
update.message = str(message)
self.log('Updating %s => %s' % (task_id, self.MESOS_STATES[state]))
self.log(' Reason: %s' % message)
driver.sendStatusUpdate(update)
def run_task():
logging.debug("Mesos Executor: requested task %s",
task.task_id.value)
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = str("running discovery task")
logging.debug(update.data)
driver.sendStatusUpdate(update)
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
## NB: TODO test port availability...
update.data = str(dumps(get_telemetry(), indent=4))
## NB: TODO download tarball/container for service launch
# notify scheduler: ready to launch service
logging.debug(update.data)
driver.sendStatusUpdate(update)
def run_task():
task_path = task.task_id.value
log("Running task %s" % (task_path))
work_path = os.path.join(self.basedir, rePathClean.sub("", task_path))
update = mesos_pb2.TaskStatus()
update.task_id.value = task_path
update.state = mesos_pb2.TASK_RUNNING
update.data = str(task_path)
driver.sendStatusUpdate(update)
fail = True
try:
handle = open(work_path + TARGET_SUFFIX)
obj = pickle.loads(handle.read())
handle.close()
obj.__manager__ = TargetManager()
obj.run()
if len(obj.__manager__.child_list):
child_dir = os.path.join(work_path + CHILDREN_SUFFIX)
if not os.path.exists(child_dir):
os.mkdir(child_dir)
for name, target in obj.__manager__.child_list:
target_path = os.path.join(child_dir, name + TARGET_SUFFIX)
handle = open(target_path, "w")
handle.write(pickle.dumps(target))
handle.close()
if len(obj.__manager__.follow_list):
follow_dir = os.path.join(work_path + FOLLOW_SUFFIX)
if not os.path.exists(follow_dir):
os.mkdir(follow_dir)
for name, target in obj.__manager__.follow_list:
target_path = os.path.join(follow_dir, name + TARGET_SUFFIX)
handle = open(target_path, "w")
handle.write(pickle.dumps(target))
handle.close()
fail = False
except Exception, e:
log(str(e))
def set_error(self):
update = mesos_pb2.TaskStatus()
update.task_id.value = self.task.task_id.value
update.data = str(self.task_data['task_id'])
update.state = mesos_pb2.TASK_FAILED
self.driver.sendStatusUpdate(update)
def reply_status(driver, task, status):
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = status
driver.sendStatusUpdate(update)
