def nds_loop(doInit): ndutil.setTimezone() ndlCom = NdlCom('nDroid-Storage', '127.0.0.1', 12322) ndlCom.doCom('Initiating') ndlCom.doCom('Loading Configuration') cnfManager = CnfManager() cnfManager.load('./nds.cnf') cnfData = cnfManager.getCnfData() ndutil.enableDir(cnfData['storageDir']) ndlCom.doCom('Connecting to DB') dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName']) if doInit: dbManager.create_table() os.system('rm -fr %s/*' % cnfData['storageDir']) msgManager = MsgManager() netManager = NetManager() netManager.setStorageDir(cnfData['storageDir']) netManager.setFileMode(cnfData['fileMode']) netManager.setNdlCom(ndlCom) netManager.setDbManager(dbManager) netManager.setMsgManager(msgManager) reactor.listenUDP(cnfData['comPort'], netManager) ndlCom.doCom('Listening Com Port') reactor.run()
def ncd_loop(doInit, dlThreadNum): ndutil.setTimezone() #read config cnfManager = CnfManager() cnfManager.load('./ndc.cnf') cnfData = cnfManager.getCnfData() #check dirs ndutil.enableDir(cnfData['dirWorking']) ndutil.enableDir(cnfData['dirStore']) #ndlcom logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort']) logger.logger('Initiating') dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName']) if doInit: dbManager.create_table() os.system('rm -f %s/*' % cnfData['dirWorking']) os.system('rm -f %s/*' % cnfData['dirStore']) #logger.logger('Customizing Spiders') #spiderGenerator = SpiderGenerator('template', 'spider/spiders') #for spider in cnfData['spiders']: # spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage']) rpQueue = Queue() pdQueue = Queue() dpQueue = Queue() pdLock = threading.Lock() rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor') rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum) rpcMonitor.setDownloadTotal(pdQueue.qsize()) rpcMonitor.setPdQueueSize(pdQueue.qsize()) botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler') receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver') preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor') downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader') processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor') logger.logger('Starting Threads') rpcMonitor.start() botScheduler.start() receiver.start() preProcessor.start() downloader.start() processor.start() processor.join() downloader.join() preProcessor.join() receiver.join() botScheduler.join() rpcMonitor.join()
def enable_storage_dir(self, storageDir, uid): dirPath = '%s/%s/%s/%s/%s' % ( storageDir, uid[0:4], uid[4:8], uid[8:12], uid[12:16]) ndutil.enableDir(dirPath) return dirPath