forked from mesosphere/RENDLER
/
rendler.py
213 lines (174 loc) · 7.13 KB
/
rendler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
#!/usr/bin/env python
from collections import deque
import json
import os
import signal
import sys
import time
from threading import Thread
import mesos
import mesos_pb2
import results
import task_state
import export_dot
TASK_CPUS = 0.1
TASK_MEM = 32
# See the Mesos Framework Development Guide:
# http://mesos.apache.org/documentation/latest/app-framework-development-guide
class RenderingCrawler(mesos.Scheduler):
def __init__(self, seedUrl, crawlExecutor, renderExecutor):
print "RENDLER"
print "======="
print "seedUrl: [%s]\n" % seedUrl
self.seedUrl = seedUrl
self.crawlExecutor = crawlExecutor
self.renderExecutor = renderExecutor
self.crawlQueue = deque([seedUrl])
self.renderQueue = deque([seedUrl])
self.processedURLs = set([seedUrl])
self.crawlResults = set([])
self.renderResults = {}
self.tasksCreated = 0
self.tasksRunning = 0
self.shuttingDown = False
def registered(self, driver, frameworkId, masterInfo):
print "Registered with framework ID [%s]" % frameworkId.value
def makeTaskPrototype(self, offer):
task = mesos_pb2.TaskInfo()
tid = self.tasksCreated
self.tasksCreated += 1
task.task_id.value = str(tid)
task.slave_id.value = offer.slave_id.value
cpus = task.resources.add()
cpus.name = "cpus"
cpus.type = mesos_pb2.Value.SCALAR
cpus.scalar.value = TASK_CPUS
mem = task.resources.add()
mem.name = "mem"
mem.type = mesos_pb2.Value.SCALAR
mem.scalar.value = TASK_MEM
return task
def makeCrawlTask(self, url, offer):
task = self.makeTaskPrototype(offer)
task.name = "crawl_%s" % task.task_id
task.executor.MergeFrom(self.crawlExecutor)
task.data = str(url)
return task
def makeRenderTask(self, url, offer):
task = self.makeTaskPrototype(offer)
task.name = "render_%s" % task.task_id
task.executor.MergeFrom(self.renderExecutor)
task.data = str(url)
return task
def printQueueStatistics(self):
print "Crawl queue length: %d, Render queue length: %d, Running tasks: %d" % (
len(self.crawlQueue), len(self.renderQueue), self.tasksRunning
)
def maxTasksForOffer(self, offer):
count = 0
cpus = next(rsc.scalar.value for rsc in offer.resources if rsc.name == "cpus")
mem = next(rsc.scalar.value for rsc in offer.resources if rsc.name == "mem")
while cpus >= TASK_CPUS and mem >= TASK_MEM:
count += 1
cpus -= TASK_CPUS
mem -= TASK_MEM
return count
def resourceOffers(self, driver, offers):
self.printQueueStatistics()
for offer in offers:
print "Got resource offer [%s]" % offer.id.value
if self.shuttingDown:
print "Shutting down: declining offer on [%s]" % offer.hostname
driver.declineOffer(offer.id)
continue
maxTasks = self.maxTasksForOffer(offer)
print "maxTasksForOffer: [%d]" % maxTasks
tasks = []
for i in range(maxTasks / 2):
if self.crawlQueue:
crawlTaskUrl = self.crawlQueue.popleft()
task = self.makeCrawlTask(crawlTaskUrl, offer)
tasks.append(task)
if self.renderQueue:
renderTaskUrl = self.renderQueue.popleft()
task = self.makeRenderTask(renderTaskUrl, offer)
tasks.append(task)
if tasks:
print "Accepting offer on [%s]" % offer.hostname
driver.launchTasks(offer.id, tasks)
else:
print "Declining offer on [%s]" % offer.hostname
driver.declineOffer(offer.id)
def statusUpdate(self, driver, update):
stateName = task_state.nameFor[update.state]
print "Task [%s] is in state [%s]" % (update.task_id.value, stateName)
if update.state == 1: # Running
self.tasksRunning += 1
elif self.tasksRunning > 0 and update.state > 1: # Terminal state
self.tasksRunning -= 1
def frameworkMessage(self, driver, executorId, slaveId, message):
o = json.loads(message)
if executorId.value == crawlExecutor.executor_id.value:
result = results.CrawlResult(o['taskId'], o['url'], o['links'])
for link in result.links:
edge = (result.url, link)
print "Appending [%s] to crawl results" % repr(edge)
self.crawlResults.add(edge)
if not link in self.processedURLs:
print "Enqueueing [%s]" % link
self.crawlQueue.append(link)
self.renderQueue.append(link)
self.processedURLs.add(link)
elif executorId.value == renderExecutor.executor_id.value:
result = results.RenderResult(o['taskId'], o['url'], o['imageUrl'])
print "Appending [%s] to render results" % repr((result.url, result.imageUrl))
self.renderResults[result.url] = result.imageUrl
def shutdown(signal, frame):
print "Rendler is shutting down"
rendler.shuttingDown = True
while rendler.tasksRunning > 0:
time.sleep(1)
driver.stop()
export_dot.dot(rendler.crawlResults, rendler.renderResults, "result.dot")
print "Goodbye!"
sys.exit(0)
if __name__ == "__main__":
if len(sys.argv) != 3:
print "Usage: %s seedUrl mesosMasterUrl" % sys.argv[0]
sys.exit(1)
uris = [ "crawl_executor.py",
"export_dot.py",
"render.js",
"render_executor.py",
"results.py",
"task_state.py" ]
crawlExecutor = mesos_pb2.ExecutorInfo()
crawlExecutor.executor_id.value = "crawl-executor"
crawlExecutor.command.value = "python crawl_executor.py"
for uri in uris:
uri_proto = crawlExecutor.command.uris.add()
uri_proto.value = "/home/vagrant/hostfiles/" + uri
uri_proto.extract = False
crawlExecutor.name = "Crawler"
renderExecutor = mesos_pb2.ExecutorInfo()
renderExecutor.executor_id.value = "render-executor"
renderExecutor.command.value = "python render_executor.py --local"
for uri in uris:
uri_proto = renderExecutor.command.uris.add()
uri_proto.value = "/home/vagrant/hostfiles/" + uri
uri_proto.extract = False
renderExecutor.name = "Renderer"
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "RENDLER"
rendler = RenderingCrawler(sys.argv[1], crawlExecutor, renderExecutor)
driver = mesos.MesosSchedulerDriver(rendler, framework, sys.argv[2])
# driver.run() blocks; we run it in a separate thread
def run_driver_async():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop()
sys.exit(status)
Thread(target = run_driver_async, args = ()).start()
print "(Listening for Ctrl-C)"
signal.signal(signal.SIGINT, shutdown)
while True: time.sleep(1)