def nda_loop():
ndutil.setTimezone()
ndlCom = NdlCom('nDroid-Executer', '127.0.0.1', 12322)
ndlCom.doCom('Initiating')
ndlCom.doCom('Loading Config')
cnfManager = CnfManager()
cnfManager.load('./nde.cnf')
cnfData = cnfManager.getCnfData()
nsQueue = Queue()
nsLock = threading.Lock()
netManager = NetManager()
netManager.setNdlCom(ndlCom)
netManager.setNsQueue(nsQueue, nsLock)
ndlCom.doCom('Starting Threads')
scheduler = Scheduler([ndlCom, nsQueue, nsLock], 'Scheduler')
scheduler.start()
reactor.listenUDP(cnfData['comPort'], netManager)
ndlCom.doCom('Listening Com Port')
reactor.run()
scheduler.join()
def nds_loop(doInit):
ndutil.setTimezone()
ndlCom = NdlCom('nDroid-Storage', '127.0.0.1', 12322)
ndlCom.doCom('Initiating')
ndlCom.doCom('Loading Configuration')
cnfManager = CnfManager()
cnfManager.load('./nds.cnf')
cnfData = cnfManager.getCnfData()
ndutil.enableDir(cnfData['storageDir'])
ndlCom.doCom('Connecting to DB')
dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
if doInit:
dbManager.create_table()
os.system('rm -fr %s/*' % cnfData['storageDir'])
msgManager = MsgManager()
netManager = NetManager()
netManager.setStorageDir(cnfData['storageDir'])
netManager.setFileMode(cnfData['fileMode'])
netManager.setNdlCom(ndlCom)
netManager.setDbManager(dbManager)
netManager.setMsgManager(msgManager)
reactor.listenUDP(cnfData['comPort'], netManager)
ndlCom.doCom('Listening Com Port')
reactor.run()
def ncd_loop(doInit, dlThreadNum):
ndutil.setTimezone()
#read config
cnfManager = CnfManager()
cnfManager.load('./ndc.cnf')
cnfData = cnfManager.getCnfData()
#check dirs
ndutil.enableDir(cnfData['dirWorking'])
ndutil.enableDir(cnfData['dirStore'])
#ndlcom
logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort'])
logger.logger('Initiating')
dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
if doInit:
dbManager.create_table()
os.system('rm -f %s/*' % cnfData['dirWorking'])
os.system('rm -f %s/*' % cnfData['dirStore'])
#logger.logger('Customizing Spiders')
#spiderGenerator = SpiderGenerator('template', 'spider/spiders')
#for spider in cnfData['spiders']:
# spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage'])
rpQueue = Queue()
pdQueue = Queue()
dpQueue = Queue()
pdLock = threading.Lock()
rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor')
rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum)
rpcMonitor.setDownloadTotal(pdQueue.qsize())
rpcMonitor.setPdQueueSize(pdQueue.qsize())
botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler')
receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver')
preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor')
downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader')
processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor')
logger.logger('Starting Threads')
rpcMonitor.start()
botScheduler.start()
receiver.start()
preProcessor.start()
downloader.start()
processor.start()
processor.join()
downloader.join()
preProcessor.join()
receiver.join()
botScheduler.join()
rpcMonitor.join()
def ndl_loop():
ndutil.setTimezone()
cnfManager = CnfManager()
cnfManager.load('./ndl.cnf')
cnfData = cnfManager.getCnfData()
logManager = LogManager(cnfData['mode'], cnfData['logFile'])
logManager.intLog('Iniatiating')
netManager = NetManager()
netManager.setLogManager(logManager)
reactor.listenUDP(cnfData['port'], netManager)
logManager.intLog('Listening Com Port')
reactor.run()
def ndc_loop():
ndutil.setTimezone()
logger = Logger('nDroid-CD', '127.0.0.1', 12322)
logger.logger('Initiating')
logger.logger('Loading Config')
cnfManager = CnfManager()
cnfManager.load('./ndc.cnf')
cnfData = cnfManager.getCnfData()
fsQueue = Queue()
fsLock = threading.Lock()
sfQueue = Queue()
sfLock = threading.Lock()
logger.logger('Connecting to DB')
dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
dbManager.create_table()
netManager = NetManager()
netManager.setNdlCom(logger)
netManager.setDbManager(dbManager)
netManager.setFsQueue(fsQueue, fsLock)
logger.logger('Starting Threads')
fileSender = FileSender([logger, fsQueue, fsLock, sfQueue, sfLock, cnfData['host'], cnfData['fileUrl'], cnfData['apiKey'], cnfData['interval']], 'FileSender')
scanSender = ScanSender([logger, fsQueue, fsLock, sfQueue, sfLock, cnfData['host'], cnfData['scanUrl'], cnfData['apiKey'], cnfData['interval'], dbManager], 'ScanSender')
fileSender.start()
scanSender.start()
reactor.listenUDP(cnfData['comPort'], netManager)
logger.logger('Listening Com Port')
reactor.run()
scanSender.join()
fileSender.join()
def nda_loop(doInit):
ndutil.setTimezone()
ndlCom = NdlCom('nDroid-ADB', '127.0.0.1', 12322)
ndlCom.doCom('Initiating')
ndlCom.doCom('Loading Config')
cnfManager = CnfManager()
cnfManager.load('./nda.cnf')
cnfData = cnfManager.get_cnf_data()
ndlCom.doCom('Connecting to DB')
dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
if doInit:
dbManager.create_table()
netManager = NetManager()
netManager.setNdlCom(ndlCom)
netManager.setDbManager(dbManager)
reactor.listenUDP(cnfData['comPort'], netManager)
ndlCom.doCom('Listening Com Port')
reactor.run()