def frameworkMessage(self, driver, executorId, slaveId, message):
"""
Invoked when an executor sends a message. These messages are best
effort; do not expect a framework message to be retransmitted in any
reliable fashion.
"""
o = json.loads(message)
if executorId.value == crawlExecutor.executor_id.value:
result = results.CrawlResult(o['taskId'], o['url'], o['links'])
#
# TODO
#
elif executorId.value == renderExecutor.executor_id.value:
result = results.RenderResult(o['taskId'], o['url'], o['imageUrl'])
def run_task():
print "Running render task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
driver.sendStatusUpdate(update)
url = task.data
# 1) Render picture to hash file name.
destination = uuid.uuid4().hex + ".png"
if call(["phantomjs", "render.js", url, destination]) != 0:
print "Could not render " + url
return
if not self.local:
# 2) Upload to s3.
remote_destination = "s3://downloads.mesosphere.io/demo/artifacts/" + destination
print "Uploading image to " + remote_destination
if call(["s3cmd", "put", destination, remote_destination]) != 0:
print "Could not upload " + destination + " to " + remote_destination
return
else:
remote_destination = "file:///" + os.getcwd() + "/" + destination
# 3) Announce render result to framework.
print "Announcing render result"
res = results.RenderResult(
task.task_id.value,
url,
remote_destination
)
message = repr(res)
driver.sendFrameworkMessage(message)
print "Sending status update for task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update for task %s" % task.task_id.value
return
def frameworkMessage(self, driver, executorId, slaveId, message):
o = json.loads(message)
if executorId.value == crawlExecutor.executor_id.value:
result = results.CrawlResult(o['taskId'], o['url'], o['links'])
for link in result.links:
edge = (result.url, link)
print "Appending [%s] to crawl results" % repr(edge)
self.crawlResults.add(edge)
if not link in self.processedURLs:
print "Enqueueing [%s]" % link
self.crawlQueue.append(link)
self.renderQueue.append(link)
self.processedURLs.add(link)
elif executorId.value == renderExecutor.executor_id.value:
result = results.RenderResult(o['taskId'], o['url'], o['imageUrl'])
print "Appending [%s] to render results" % repr(
(result.url, result.imageUrl))
self.renderResults[result.url] = result.imageUrl
def frameworkMessage(self, driver, executorId, slaveId, message):
o = json.loads(message)
if executorId.value == crawlExecutor.executor_id.value:
result = results.CrawlResult(o['taskId'], o['url'], o['links'])
for link in result.links:
edge = (result.url, link)
print "Appending [%s] to crawl results" % repr(edge)
self.crawlResults.add(edge)
if not self.renderLimitReached and self.maxRenderTasks > 0 and \
self.maxRenderTasks <= len(self.processedURLs):
print "Render task limit (%d) reached" % self.maxRenderTasks
self.renderLimitReached = True
if not link in self.processedURLs and not self.renderLimitReached:
print "Enqueueing [%s]" % link
self.crawlQueue.append(link)
self.renderQueue.append(link)
self.processedURLs.add(link)
elif executorId.value == renderExecutor.executor_id.value:
result = results.RenderResult(o['taskId'], o['url'], o['imageUrl'])
print "Appending [%s] to render results" % repr(
(result.url, result.imageUrl))
self.renderResults[result.url] = result.imageUrl