def ncd_loop(doInit, dlThreadNum):
ndutil.setTimezone()
#read config
cnfManager = CnfManager()
cnfManager.load('./ndc.cnf')
cnfData = cnfManager.getCnfData()
#check dirs
ndutil.enableDir(cnfData['dirWorking'])
ndutil.enableDir(cnfData['dirStore'])
#ndlcom
logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort'])
logger.logger('Initiating')
dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
if doInit:
dbManager.create_table()
os.system('rm -f %s/*' % cnfData['dirWorking'])
os.system('rm -f %s/*' % cnfData['dirStore'])
#logger.logger('Customizing Spiders')
#spiderGenerator = SpiderGenerator('template', 'spider/spiders')
#for spider in cnfData['spiders']:
# spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage'])
rpQueue = Queue()
pdQueue = Queue()
dpQueue = Queue()
pdLock = threading.Lock()
rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor')
rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum)
rpcMonitor.setDownloadTotal(pdQueue.qsize())
rpcMonitor.setPdQueueSize(pdQueue.qsize())
botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler')
receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver')
preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor')
downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader')
processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor')
logger.logger('Starting Threads')
rpcMonitor.start()
botScheduler.start()
receiver.start()
preProcessor.start()
downloader.start()
processor.start()
processor.join()
downloader.join()
preProcessor.join()
receiver.join()
botScheduler.join()
rpcMonitor.join()
