Beispiel #1
0
def nda_loop():
	ndutil.setTimezone()

	ndlCom = NdlCom('nDroid-Executer', '127.0.0.1', 12322)
	ndlCom.doCom('Initiating')

	ndlCom.doCom('Loading Config')
	cnfManager = CnfManager()
	cnfManager.load('./nde.cnf')
	cnfData = cnfManager.getCnfData()

	nsQueue = Queue()
	nsLock = threading.Lock()

	netManager = NetManager()
	netManager.setNdlCom(ndlCom)
	netManager.setNsQueue(nsQueue, nsLock)

	ndlCom.doCom('Starting Threads')
	scheduler = Scheduler([ndlCom, nsQueue, nsLock], 'Scheduler')

	scheduler.start()

	reactor.listenUDP(cnfData['comPort'], netManager)
	ndlCom.doCom('Listening Com Port')
	reactor.run()

	scheduler.join()
Beispiel #2
0
def nds_loop(doInit):
	ndutil.setTimezone()

	ndlCom = NdlCom('nDroid-Storage', '127.0.0.1', 12322)
	ndlCom.doCom('Initiating')

	ndlCom.doCom('Loading Configuration')
	cnfManager = CnfManager()
	cnfManager.load('./nds.cnf')
	cnfData = cnfManager.getCnfData()

	ndutil.enableDir(cnfData['storageDir'])

	ndlCom.doCom('Connecting to DB')
	dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
	if doInit:
		dbManager.create_table()
		os.system('rm -fr %s/*' % cnfData['storageDir'])

	msgManager = MsgManager()

	netManager = NetManager()
	netManager.setStorageDir(cnfData['storageDir'])
	netManager.setFileMode(cnfData['fileMode'])
	netManager.setNdlCom(ndlCom)
	netManager.setDbManager(dbManager)
	netManager.setMsgManager(msgManager)

	reactor.listenUDP(cnfData['comPort'], netManager)
	ndlCom.doCom('Listening Com Port')
	reactor.run()
Beispiel #3
0
def ncd_loop(doInit, dlThreadNum):
	ndutil.setTimezone()

#read config
	cnfManager = CnfManager()
	cnfManager.load('./ndc.cnf')
	cnfData = cnfManager.getCnfData()

#check dirs
	ndutil.enableDir(cnfData['dirWorking'])
	ndutil.enableDir(cnfData['dirStore'])

#ndlcom
	logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort'])
	logger.logger('Initiating')

	dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
	if doInit:
		dbManager.create_table()
		os.system('rm -f %s/*' % cnfData['dirWorking'])
		os.system('rm -f %s/*' % cnfData['dirStore'])

	#logger.logger('Customizing Spiders')
	#spiderGenerator = SpiderGenerator('template', 'spider/spiders')
	#for spider in cnfData['spiders']:
	#	spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage'])

	rpQueue = Queue()
	pdQueue = Queue()
	dpQueue = Queue()
	pdLock = threading.Lock()

	rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor')
	rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum)
	rpcMonitor.setDownloadTotal(pdQueue.qsize())
	rpcMonitor.setPdQueueSize(pdQueue.qsize())
	
	botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler')
	receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver')
	preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor')
	downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader')
	processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor')

	logger.logger('Starting Threads')
	rpcMonitor.start()
	botScheduler.start()
	receiver.start()
	preProcessor.start()
	downloader.start()
	processor.start()
	
	processor.join()
	downloader.join()
	preProcessor.join()
	receiver.join()
	botScheduler.join()
	rpcMonitor.join()
Beispiel #4
0
def ndl_loop():
	ndutil.setTimezone()

	cnfManager = CnfManager()
	cnfManager.load('./ndl.cnf')
	cnfData = cnfManager.getCnfData()

	logManager = LogManager(cnfData['mode'], cnfData['logFile'])
	logManager.intLog('Iniatiating')

	netManager = NetManager()
	netManager.setLogManager(logManager)

	reactor.listenUDP(cnfData['port'], netManager)
	logManager.intLog('Listening Com Port')
	reactor.run()
Beispiel #5
0
def ndc_loop():
	ndutil.setTimezone()

	logger = Logger('nDroid-CD', '127.0.0.1', 12322)
	logger.logger('Initiating')

	logger.logger('Loading Config')
	cnfManager = CnfManager()
	cnfManager.load('./ndc.cnf')
	cnfData = cnfManager.getCnfData()

	fsQueue = Queue()
	fsLock = threading.Lock()
	sfQueue = Queue()
	sfLock = threading.Lock()

	logger.logger('Connecting to DB')
	dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
	dbManager.create_table()

	netManager = NetManager()
	netManager.setNdlCom(logger)
	netManager.setDbManager(dbManager)
	netManager.setFsQueue(fsQueue, fsLock)

	logger.logger('Starting Threads')
	fileSender = FileSender([logger, fsQueue, fsLock, sfQueue, sfLock, cnfData['host'], cnfData['fileUrl'], cnfData['apiKey'], cnfData['interval']], 'FileSender')
	scanSender = ScanSender([logger, fsQueue, fsLock, sfQueue, sfLock, cnfData['host'], cnfData['scanUrl'], cnfData['apiKey'], cnfData['interval'], dbManager], 'ScanSender')

	fileSender.start()
	scanSender.start()

	reactor.listenUDP(cnfData['comPort'], netManager)
	logger.logger('Listening Com Port')
	reactor.run()

	scanSender.join()
	fileSender.join()
Beispiel #6
0
def nda_loop(doInit):
    ndutil.setTimezone()

    ndlCom = NdlCom('nDroid-ADB', '127.0.0.1', 12322)
    ndlCom.doCom('Initiating')

    ndlCom.doCom('Loading Config')
    cnfManager = CnfManager()
    cnfManager.load('./nda.cnf')
    cnfData = cnfManager.get_cnf_data()

    ndlCom.doCom('Connecting to DB')
    dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])

    if doInit:
        dbManager.create_table()

    netManager = NetManager()
    netManager.setNdlCom(ndlCom)
    netManager.setDbManager(dbManager)

    reactor.listenUDP(cnfData['comPort'], netManager)
    ndlCom.doCom('Listening Com Port')
    reactor.run()