-
Notifications
You must be signed in to change notification settings - Fork 3
/
crawlerWorker.py
executable file
·26 lines (22 loc) · 1012 Bytes
/
crawlerWorker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#!/usr/bin/python
import sys
from optparse import OptionParser
from core.engine import Engine, MAX_URL_QSIZE, DEFAULT_REG_PORT, DEFAULT_MANAGER, DEFAULT_DOWNLOADERS
def parseCommandLineArgs():
parser = OptionParser()
parser.add_option("-m", "--manager", dest="manager", default=DEFAULT_MANAGER,
help="the name/ip of the host on which manager is started.")
parser.add_option("-p", "--port", dest="regPort", default=DEFAULT_REG_PORT,
help="port to connect manager.")
parser.add_option("-d", "--download", dest="downloaders", default=DEFAULT_DOWNLOADERS,
help="number of threads which download web pages. 4 by default.")
(options, args) = parser.parse_args()
return options.manager, int(options.regPort), int(options.downloaders)
def main():
manager, port, downloaders = parseCommandLineArgs()
engine = Engine(downloaders, manager, port)
engine.start()
raw_input("press any key to stop....\n")
engine.stop()
if __name__ == "__main__":
main()