def post(self): if not request.json: return {'message': 'Problems parsing JSON'}, 400 if request.json.get('urls', None) is None: return {'message': 'Validation Failed', 'errors': [{'resource:': 'Package', 'field': 'urls', 'code': 'missing_field'}]}, 422 if not isinstance(request.json['urls'], list): return {'message': 'Validation Failed', 'errors': [{'resource:': 'Package', 'field': 'urls', 'code': 'invalid'}]}, 422 try: parent_conn, child_conn = multiprocessing.Pipe() d = Downloader(child_conn, app.config['BASEPATH'], request.json['urls']) d.start() r = parent_conn.recv() d.join() p = Package(ip=request.remote_addr, path=r['zipname'], date_created=arrow.now().datetime) db.session.add(p) db.session.commit() except Exception as e: logger.error(e) return {'package': 'http://{0}/{1}'.format( app.config['SERVER_ADDR'], r['name'])}, 201
def ncd_loop(doInit, dlThreadNum): ndutil.setTimezone() #read config cnfManager = CnfManager() cnfManager.load('./ndc.cnf') cnfData = cnfManager.getCnfData() #check dirs ndutil.enableDir(cnfData['dirWorking']) ndutil.enableDir(cnfData['dirStore']) #ndlcom logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort']) logger.logger('Initiating') dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName']) if doInit: dbManager.create_table() os.system('rm -f %s/*' % cnfData['dirWorking']) os.system('rm -f %s/*' % cnfData['dirStore']) #logger.logger('Customizing Spiders') #spiderGenerator = SpiderGenerator('template', 'spider/spiders') #for spider in cnfData['spiders']: # spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage']) rpQueue = Queue() pdQueue = Queue() dpQueue = Queue() pdLock = threading.Lock() rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor') rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum) rpcMonitor.setDownloadTotal(pdQueue.qsize()) rpcMonitor.setPdQueueSize(pdQueue.qsize()) botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler') receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver') preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor') downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader') processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor') logger.logger('Starting Threads') rpcMonitor.start() botScheduler.start() receiver.start() preProcessor.start() downloader.start() processor.start() processor.join() downloader.join() preProcessor.join() receiver.join() botScheduler.join() rpcMonitor.join()
def run(self): print 'ENGINE STSRT' self.is_running=True for i in range(settings.S_thre_num): t=Downloader(i,self.scheduler,self) self.threads.append(t) for t in self.threads: t.start() time.sleep(10) for t in self.threads: t.join() self.stop()